diff options
364 files changed, 40054 insertions, 20 deletions
diff --git a/external_libs/cpp/json/json/json-forwards.h b/external_libs/json/json/json-forwards.h index ccbdb2b1..ccbdb2b1 100644 --- a/external_libs/cpp/json/json/json-forwards.h +++ b/external_libs/json/json/json-forwards.h diff --git a/external_libs/cpp/json/json/json.h b/external_libs/json/json/json.h index e01991e0..e01991e0 100644 --- a/external_libs/cpp/json/json/json.h +++ b/external_libs/json/json/json.h diff --git a/external_libs/cpp/json/jsoncpp.cpp b/external_libs/json/jsoncpp.cpp index e9f7e986..e9f7e986 100644 --- a/external_libs/cpp/json/jsoncpp.cpp +++ b/external_libs/json/jsoncpp.cpp diff --git a/external_libs/cpp/yaml-cpp/CMakeLists.txt b/external_libs/yaml-cpp/CMakeLists.txt index 823ce201..823ce201 100755..100644 --- a/external_libs/cpp/yaml-cpp/CMakeLists.txt +++ b/external_libs/yaml-cpp/CMakeLists.txt diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/aliasmanager.h b/external_libs/yaml-cpp/include/yaml-cpp/aliasmanager.h index e90c93dd..e90c93dd 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/aliasmanager.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/aliasmanager.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/anchor.h b/external_libs/yaml-cpp/include/yaml-cpp/anchor.h index 433f2fa5..433f2fa5 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/anchor.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/anchor.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/binary.h b/external_libs/yaml-cpp/include/yaml-cpp/binary.h index 8504ebeb..8504ebeb 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/binary.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/binary.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/contrib/anchordict.h b/external_libs/yaml-cpp/include/yaml-cpp/contrib/anchordict.h index e483dc4b..e483dc4b 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/contrib/anchordict.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/contrib/anchordict.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/contrib/graphbuilder.h b/external_libs/yaml-cpp/include/yaml-cpp/contrib/graphbuilder.h index 6739a12b..6739a12b 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/contrib/graphbuilder.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/contrib/graphbuilder.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/conversion.h b/external_libs/yaml-cpp/include/yaml-cpp/conversion.h index 1b557b56..1b557b56 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/conversion.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/conversion.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/dll.h b/external_libs/yaml-cpp/include/yaml-cpp/dll.h index ea138401..ea138401 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/dll.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/dll.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/emitfromevents.h b/external_libs/yaml-cpp/include/yaml-cpp/emitfromevents.h index e11ae640..e11ae640 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/emitfromevents.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/emitfromevents.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/emitter.h b/external_libs/yaml-cpp/include/yaml-cpp/emitter.h index 1d7edf2f..1d7edf2f 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/emitter.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/emitter.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/emittermanip.h b/external_libs/yaml-cpp/include/yaml-cpp/emittermanip.h index a8ec64a4..a8ec64a4 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/emittermanip.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/emittermanip.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/eventhandler.h b/external_libs/yaml-cpp/include/yaml-cpp/eventhandler.h index 3173a1fb..3173a1fb 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/eventhandler.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/eventhandler.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/exceptions.h b/external_libs/yaml-cpp/include/yaml-cpp/exceptions.h index 394d5868..394d5868 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/exceptions.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/exceptions.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/iterator.h b/external_libs/yaml-cpp/include/yaml-cpp/iterator.h index 400ee340..400ee340 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/iterator.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/iterator.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/ltnode.h b/external_libs/yaml-cpp/include/yaml-cpp/ltnode.h index 30b4f950..30b4f950 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/ltnode.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/ltnode.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/mark.h b/external_libs/yaml-cpp/include/yaml-cpp/mark.h index 7c80fbcb..7c80fbcb 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/mark.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/mark.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/node.h b/external_libs/yaml-cpp/include/yaml-cpp/node.h index e78190e0..e78190e0 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/node.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/node.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/nodeimpl.h b/external_libs/yaml-cpp/include/yaml-cpp/nodeimpl.h index 5ca7ddba..5ca7ddba 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/nodeimpl.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/nodeimpl.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/nodereadimpl.h b/external_libs/yaml-cpp/include/yaml-cpp/nodereadimpl.h index 6838dc5a..6838dc5a 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/nodereadimpl.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/nodereadimpl.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/nodeutil.h b/external_libs/yaml-cpp/include/yaml-cpp/nodeutil.h index d0c01d27..d0c01d27 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/nodeutil.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/nodeutil.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/noncopyable.h b/external_libs/yaml-cpp/include/yaml-cpp/noncopyable.h index 8e61e433..8e61e433 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/noncopyable.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/noncopyable.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/null.h b/external_libs/yaml-cpp/include/yaml-cpp/null.h index 711f18c3..711f18c3 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/null.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/null.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/ostream.h b/external_libs/yaml-cpp/include/yaml-cpp/ostream.h index 65839b1b..65839b1b 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/ostream.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/ostream.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/parser.h b/external_libs/yaml-cpp/include/yaml-cpp/parser.h index f71cdff4..f71cdff4 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/parser.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/parser.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/stlemitter.h b/external_libs/yaml-cpp/include/yaml-cpp/stlemitter.h index f8ff20ea..f8ff20ea 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/stlemitter.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/stlemitter.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/stlnode.h b/external_libs/yaml-cpp/include/yaml-cpp/stlnode.h index 40d4ae79..40d4ae79 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/stlnode.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/stlnode.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/traits.h b/external_libs/yaml-cpp/include/yaml-cpp/traits.h index 09eead44..09eead44 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/traits.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/traits.h diff --git a/external_libs/cpp/yaml-cpp/include/yaml-cpp/yaml.h b/external_libs/yaml-cpp/include/yaml-cpp/yaml.h index 29595553..29595553 100755..100644 --- a/external_libs/cpp/yaml-cpp/include/yaml-cpp/yaml.h +++ b/external_libs/yaml-cpp/include/yaml-cpp/yaml.h diff --git a/external_libs/cpp/yaml-cpp/install.txt b/external_libs/yaml-cpp/install.txt index 93923624..93923624 100755..100644 --- a/external_libs/cpp/yaml-cpp/install.txt +++ b/external_libs/yaml-cpp/install.txt diff --git a/external_libs/cpp/yaml-cpp/license.txt b/external_libs/yaml-cpp/license.txt index 5bd9e1a1..5bd9e1a1 100755..100644 --- a/external_libs/cpp/yaml-cpp/license.txt +++ b/external_libs/yaml-cpp/license.txt diff --git a/external_libs/cpp/yaml-cpp/src/aliasmanager.cpp b/external_libs/yaml-cpp/src/aliasmanager.cpp index ed4d3b5a..ed4d3b5a 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/aliasmanager.cpp +++ b/external_libs/yaml-cpp/src/aliasmanager.cpp diff --git a/external_libs/cpp/yaml-cpp/src/binary.cpp b/external_libs/yaml-cpp/src/binary.cpp index 589eb089..589eb089 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/binary.cpp +++ b/external_libs/yaml-cpp/src/binary.cpp diff --git a/external_libs/cpp/yaml-cpp/src/collectionstack.h b/external_libs/yaml-cpp/src/collectionstack.h index 4a986bc9..4a986bc9 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/collectionstack.h +++ b/external_libs/yaml-cpp/src/collectionstack.h diff --git a/external_libs/cpp/yaml-cpp/src/contrib/graphbuilder.cpp b/external_libs/yaml-cpp/src/contrib/graphbuilder.cpp index ab5159cc..ab5159cc 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/contrib/graphbuilder.cpp +++ b/external_libs/yaml-cpp/src/contrib/graphbuilder.cpp diff --git a/external_libs/cpp/yaml-cpp/src/contrib/graphbuilderadapter.cpp b/external_libs/yaml-cpp/src/contrib/graphbuilderadapter.cpp index 557e97c8..557e97c8 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/contrib/graphbuilderadapter.cpp +++ b/external_libs/yaml-cpp/src/contrib/graphbuilderadapter.cpp diff --git a/external_libs/cpp/yaml-cpp/src/contrib/graphbuilderadapter.h b/external_libs/yaml-cpp/src/contrib/graphbuilderadapter.h index 3ef8ab6c..3ef8ab6c 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/contrib/graphbuilderadapter.h +++ b/external_libs/yaml-cpp/src/contrib/graphbuilderadapter.h diff --git a/external_libs/cpp/yaml-cpp/src/conversion.cpp b/external_libs/yaml-cpp/src/conversion.cpp index f81e1a0b..f81e1a0b 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/conversion.cpp +++ b/external_libs/yaml-cpp/src/conversion.cpp diff --git a/external_libs/cpp/yaml-cpp/src/directives.cpp b/external_libs/yaml-cpp/src/directives.cpp index faf1483b..faf1483b 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/directives.cpp +++ b/external_libs/yaml-cpp/src/directives.cpp diff --git a/external_libs/cpp/yaml-cpp/src/directives.h b/external_libs/yaml-cpp/src/directives.h index a3308f72..a3308f72 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/directives.h +++ b/external_libs/yaml-cpp/src/directives.h diff --git a/external_libs/cpp/yaml-cpp/src/emitfromevents.cpp b/external_libs/yaml-cpp/src/emitfromevents.cpp index 49fc10b2..49fc10b2 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/emitfromevents.cpp +++ b/external_libs/yaml-cpp/src/emitfromevents.cpp diff --git a/external_libs/cpp/yaml-cpp/src/emitter.cpp b/external_libs/yaml-cpp/src/emitter.cpp index 91f48da7..91f48da7 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/emitter.cpp +++ b/external_libs/yaml-cpp/src/emitter.cpp diff --git a/external_libs/cpp/yaml-cpp/src/emitterstate.cpp b/external_libs/yaml-cpp/src/emitterstate.cpp index 562e82c9..562e82c9 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/emitterstate.cpp +++ b/external_libs/yaml-cpp/src/emitterstate.cpp diff --git a/external_libs/cpp/yaml-cpp/src/emitterstate.h b/external_libs/yaml-cpp/src/emitterstate.h index 5698e325..5698e325 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/emitterstate.h +++ b/external_libs/yaml-cpp/src/emitterstate.h diff --git a/external_libs/cpp/yaml-cpp/src/emitterutils.cpp b/external_libs/yaml-cpp/src/emitterutils.cpp index 3d184d6c..3d184d6c 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/emitterutils.cpp +++ b/external_libs/yaml-cpp/src/emitterutils.cpp diff --git a/external_libs/cpp/yaml-cpp/src/emitterutils.h b/external_libs/yaml-cpp/src/emitterutils.h index 0e270d69..0e270d69 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/emitterutils.h +++ b/external_libs/yaml-cpp/src/emitterutils.h diff --git a/external_libs/cpp/yaml-cpp/src/exp.cpp b/external_libs/yaml-cpp/src/exp.cpp index 7bc54546..7bc54546 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/exp.cpp +++ b/external_libs/yaml-cpp/src/exp.cpp diff --git a/external_libs/cpp/yaml-cpp/src/exp.h b/external_libs/yaml-cpp/src/exp.h index 3e12aba4..3e12aba4 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/exp.h +++ b/external_libs/yaml-cpp/src/exp.h diff --git a/external_libs/cpp/yaml-cpp/src/indentation.h b/external_libs/yaml-cpp/src/indentation.h index 25f684f8..25f684f8 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/indentation.h +++ b/external_libs/yaml-cpp/src/indentation.h diff --git a/external_libs/cpp/yaml-cpp/src/iterator.cpp b/external_libs/yaml-cpp/src/iterator.cpp index f4159e32..f4159e32 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/iterator.cpp +++ b/external_libs/yaml-cpp/src/iterator.cpp diff --git a/external_libs/cpp/yaml-cpp/src/iterpriv.h b/external_libs/yaml-cpp/src/iterpriv.h index c511e8ac..c511e8ac 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/iterpriv.h +++ b/external_libs/yaml-cpp/src/iterpriv.h diff --git a/external_libs/cpp/yaml-cpp/src/node.cpp b/external_libs/yaml-cpp/src/node.cpp index 360b4ad9..360b4ad9 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/node.cpp +++ b/external_libs/yaml-cpp/src/node.cpp diff --git a/external_libs/cpp/yaml-cpp/src/nodebuilder.cpp b/external_libs/yaml-cpp/src/nodebuilder.cpp index 13a70326..13a70326 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/nodebuilder.cpp +++ b/external_libs/yaml-cpp/src/nodebuilder.cpp diff --git a/external_libs/cpp/yaml-cpp/src/nodebuilder.h b/external_libs/yaml-cpp/src/nodebuilder.h index 9c1d16a0..9c1d16a0 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/nodebuilder.h +++ b/external_libs/yaml-cpp/src/nodebuilder.h diff --git a/external_libs/cpp/yaml-cpp/src/nodeownership.cpp b/external_libs/yaml-cpp/src/nodeownership.cpp index 118edbc8..118edbc8 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/nodeownership.cpp +++ b/external_libs/yaml-cpp/src/nodeownership.cpp diff --git a/external_libs/cpp/yaml-cpp/src/nodeownership.h b/external_libs/yaml-cpp/src/nodeownership.h index 69870814..69870814 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/nodeownership.h +++ b/external_libs/yaml-cpp/src/nodeownership.h diff --git a/external_libs/cpp/yaml-cpp/src/null.cpp b/external_libs/yaml-cpp/src/null.cpp index 08fa9aae..08fa9aae 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/null.cpp +++ b/external_libs/yaml-cpp/src/null.cpp diff --git a/external_libs/cpp/yaml-cpp/src/ostream.cpp b/external_libs/yaml-cpp/src/ostream.cpp index a7f1e14b..a7f1e14b 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/ostream.cpp +++ b/external_libs/yaml-cpp/src/ostream.cpp diff --git a/external_libs/cpp/yaml-cpp/src/parser.cpp b/external_libs/yaml-cpp/src/parser.cpp index b836823f..b836823f 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/parser.cpp +++ b/external_libs/yaml-cpp/src/parser.cpp diff --git a/external_libs/cpp/yaml-cpp/src/ptr_stack.h b/external_libs/yaml-cpp/src/ptr_stack.h index bf454fb3..bf454fb3 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/ptr_stack.h +++ b/external_libs/yaml-cpp/src/ptr_stack.h diff --git a/external_libs/cpp/yaml-cpp/src/ptr_vector.h b/external_libs/yaml-cpp/src/ptr_vector.h index 7b936cb5..7b936cb5 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/ptr_vector.h +++ b/external_libs/yaml-cpp/src/ptr_vector.h diff --git a/external_libs/cpp/yaml-cpp/src/regex.cpp b/external_libs/yaml-cpp/src/regex.cpp index b35b1f43..b35b1f43 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/regex.cpp +++ b/external_libs/yaml-cpp/src/regex.cpp diff --git a/external_libs/cpp/yaml-cpp/src/regex.h b/external_libs/yaml-cpp/src/regex.h index 8722e626..8722e626 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/regex.h +++ b/external_libs/yaml-cpp/src/regex.h diff --git a/external_libs/cpp/yaml-cpp/src/regeximpl.h b/external_libs/yaml-cpp/src/regeximpl.h index d5c20d74..d5c20d74 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/regeximpl.h +++ b/external_libs/yaml-cpp/src/regeximpl.h diff --git a/external_libs/cpp/yaml-cpp/src/scanner.cpp b/external_libs/yaml-cpp/src/scanner.cpp index 199ef25a..199ef25a 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/scanner.cpp +++ b/external_libs/yaml-cpp/src/scanner.cpp diff --git a/external_libs/cpp/yaml-cpp/src/scanner.h b/external_libs/yaml-cpp/src/scanner.h index bc8dcbe5..bc8dcbe5 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/scanner.h +++ b/external_libs/yaml-cpp/src/scanner.h diff --git a/external_libs/cpp/yaml-cpp/src/scanscalar.cpp b/external_libs/yaml-cpp/src/scanscalar.cpp index 064c0867..064c0867 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/scanscalar.cpp +++ b/external_libs/yaml-cpp/src/scanscalar.cpp diff --git a/external_libs/cpp/yaml-cpp/src/scanscalar.h b/external_libs/yaml-cpp/src/scanscalar.h index c198cb18..c198cb18 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/scanscalar.h +++ b/external_libs/yaml-cpp/src/scanscalar.h diff --git a/external_libs/cpp/yaml-cpp/src/scantag.cpp b/external_libs/yaml-cpp/src/scantag.cpp index b71cbcc4..b71cbcc4 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/scantag.cpp +++ b/external_libs/yaml-cpp/src/scantag.cpp diff --git a/external_libs/cpp/yaml-cpp/src/scantag.h b/external_libs/yaml-cpp/src/scantag.h index 38437c03..38437c03 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/scantag.h +++ b/external_libs/yaml-cpp/src/scantag.h diff --git a/external_libs/cpp/yaml-cpp/src/scantoken.cpp b/external_libs/yaml-cpp/src/scantoken.cpp index 06d9cd62..06d9cd62 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/scantoken.cpp +++ b/external_libs/yaml-cpp/src/scantoken.cpp diff --git a/external_libs/cpp/yaml-cpp/src/setting.h b/external_libs/yaml-cpp/src/setting.h index 806ccdae..806ccdae 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/setting.h +++ b/external_libs/yaml-cpp/src/setting.h diff --git a/external_libs/cpp/yaml-cpp/src/simplekey.cpp b/external_libs/yaml-cpp/src/simplekey.cpp index 857a9e0b..857a9e0b 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/simplekey.cpp +++ b/external_libs/yaml-cpp/src/simplekey.cpp diff --git a/external_libs/cpp/yaml-cpp/src/singledocparser.cpp b/external_libs/yaml-cpp/src/singledocparser.cpp index 47759c32..47759c32 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/singledocparser.cpp +++ b/external_libs/yaml-cpp/src/singledocparser.cpp diff --git a/external_libs/cpp/yaml-cpp/src/singledocparser.h b/external_libs/yaml-cpp/src/singledocparser.h index 3798dccf..3798dccf 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/singledocparser.h +++ b/external_libs/yaml-cpp/src/singledocparser.h diff --git a/external_libs/cpp/yaml-cpp/src/stream.cpp b/external_libs/yaml-cpp/src/stream.cpp index 447b67c1..447b67c1 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/stream.cpp +++ b/external_libs/yaml-cpp/src/stream.cpp diff --git a/external_libs/cpp/yaml-cpp/src/stream.h b/external_libs/yaml-cpp/src/stream.h index 87f48dc8..87f48dc8 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/stream.h +++ b/external_libs/yaml-cpp/src/stream.h diff --git a/external_libs/cpp/yaml-cpp/src/streamcharsource.h b/external_libs/yaml-cpp/src/streamcharsource.h index 21fae4e1..21fae4e1 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/streamcharsource.h +++ b/external_libs/yaml-cpp/src/streamcharsource.h diff --git a/external_libs/cpp/yaml-cpp/src/stringsource.h b/external_libs/yaml-cpp/src/stringsource.h index 21be3c9a..21be3c9a 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/stringsource.h +++ b/external_libs/yaml-cpp/src/stringsource.h diff --git a/external_libs/cpp/yaml-cpp/src/tag.cpp b/external_libs/yaml-cpp/src/tag.cpp index 82a47047..82a47047 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/tag.cpp +++ b/external_libs/yaml-cpp/src/tag.cpp diff --git a/external_libs/cpp/yaml-cpp/src/tag.h b/external_libs/yaml-cpp/src/tag.h index 5f77548d..5f77548d 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/tag.h +++ b/external_libs/yaml-cpp/src/tag.h diff --git a/external_libs/cpp/yaml-cpp/src/token.h b/external_libs/yaml-cpp/src/token.h index 9807e258..9807e258 100755..100644 --- a/external_libs/cpp/yaml-cpp/src/token.h +++ b/external_libs/yaml-cpp/src/token.h diff --git a/external_libs/cpp/yaml-cpp/test/CMakeLists.txt b/external_libs/yaml-cpp/test/CMakeLists.txt index 241c19ef..241c19ef 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/CMakeLists.txt +++ b/external_libs/yaml-cpp/test/CMakeLists.txt diff --git a/external_libs/cpp/yaml-cpp/test/emittertests.cpp b/external_libs/yaml-cpp/test/emittertests.cpp index a7fdab67..a7fdab67 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/emittertests.cpp +++ b/external_libs/yaml-cpp/test/emittertests.cpp diff --git a/external_libs/cpp/yaml-cpp/test/emittertests.h b/external_libs/yaml-cpp/test/emittertests.h index e7c6ac50..e7c6ac50 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/emittertests.h +++ b/external_libs/yaml-cpp/test/emittertests.h diff --git a/external_libs/cpp/yaml-cpp/test/main.cpp b/external_libs/yaml-cpp/test/main.cpp index 64c69f10..64c69f10 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/main.cpp +++ b/external_libs/yaml-cpp/test/main.cpp diff --git a/external_libs/cpp/yaml-cpp/test/nodetests.h b/external_libs/yaml-cpp/test/nodetests.h index 733e782e..733e782e 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/nodetests.h +++ b/external_libs/yaml-cpp/test/nodetests.h diff --git a/external_libs/cpp/yaml-cpp/test/old-api/parsertests.cpp b/external_libs/yaml-cpp/test/old-api/parsertests.cpp index de7f1238..de7f1238 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/old-api/parsertests.cpp +++ b/external_libs/yaml-cpp/test/old-api/parsertests.cpp diff --git a/external_libs/cpp/yaml-cpp/test/old-api/spectests.cpp b/external_libs/yaml-cpp/test/old-api/spectests.cpp index fb5505be..fb5505be 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/old-api/spectests.cpp +++ b/external_libs/yaml-cpp/test/old-api/spectests.cpp diff --git a/external_libs/cpp/yaml-cpp/test/parsertests.h b/external_libs/yaml-cpp/test/parsertests.h index f3de1b8c..f3de1b8c 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/parsertests.h +++ b/external_libs/yaml-cpp/test/parsertests.h diff --git a/external_libs/cpp/yaml-cpp/test/specexamples.h b/external_libs/yaml-cpp/test/specexamples.h index 4688bdcf..4688bdcf 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/specexamples.h +++ b/external_libs/yaml-cpp/test/specexamples.h diff --git a/external_libs/cpp/yaml-cpp/test/spectests.cpp b/external_libs/yaml-cpp/test/spectests.cpp index bffc5062..bffc5062 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/spectests.cpp +++ b/external_libs/yaml-cpp/test/spectests.cpp diff --git a/external_libs/cpp/yaml-cpp/test/spectests.h b/external_libs/yaml-cpp/test/spectests.h index 5246df58..5246df58 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/spectests.h +++ b/external_libs/yaml-cpp/test/spectests.h diff --git a/external_libs/cpp/yaml-cpp/test/tests.cpp b/external_libs/yaml-cpp/test/tests.cpp index 2dff6eeb..2dff6eeb 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/tests.cpp +++ b/external_libs/yaml-cpp/test/tests.cpp diff --git a/external_libs/cpp/yaml-cpp/test/tests.h b/external_libs/yaml-cpp/test/tests.h index 757dbc53..757dbc53 100755..100644 --- a/external_libs/cpp/yaml-cpp/test/tests.h +++ b/external_libs/yaml-cpp/test/tests.h diff --git a/external_libs/cpp/yaml-cpp/util/CMakeLists.txt b/external_libs/yaml-cpp/util/CMakeLists.txt index 22339f02..22339f02 100755..100644 --- a/external_libs/cpp/yaml-cpp/util/CMakeLists.txt +++ b/external_libs/yaml-cpp/util/CMakeLists.txt diff --git a/external_libs/cpp/yaml-cpp/util/api.cpp b/external_libs/yaml-cpp/util/api.cpp index e5180a8a..e5180a8a 100755..100644 --- a/external_libs/cpp/yaml-cpp/util/api.cpp +++ b/external_libs/yaml-cpp/util/api.cpp diff --git a/external_libs/cpp/yaml-cpp/util/parse.cpp b/external_libs/yaml-cpp/util/parse.cpp index d02a76a7..d02a76a7 100755..100644 --- a/external_libs/cpp/yaml-cpp/util/parse.cpp +++ b/external_libs/yaml-cpp/util/parse.cpp diff --git a/external_libs/cpp/yaml-cpp/yaml-cpp.pc.cmake b/external_libs/yaml-cpp/yaml-cpp.pc.cmake index 04d343f6..04d343f6 100755..100644 --- a/external_libs/cpp/yaml-cpp/yaml-cpp.pc.cmake +++ b/external_libs/yaml-cpp/yaml-cpp.pc.cmake diff --git a/external_libs/cpp/zmq/include/zmq.h b/external_libs/zmq/include/zmq.h index f7b10db6..f7b10db6 100755..100644 --- a/external_libs/cpp/zmq/include/zmq.h +++ b/external_libs/zmq/include/zmq.h diff --git a/external_libs/cpp/zmq/include/zmq_utils.h b/external_libs/zmq/include/zmq_utils.h index 9b14aa72..9b14aa72 100755..100644 --- a/external_libs/cpp/zmq/include/zmq_utils.h +++ b/external_libs/zmq/include/zmq_utils.h diff --git a/external_libs/cpp/zmq/libzmq.a b/external_libs/zmq/libzmq.a Binary files differindex 8c994993..8c994993 100755..100644 --- a/external_libs/cpp/zmq/libzmq.a +++ b/external_libs/zmq/libzmq.a diff --git a/external_libs/cpp/zmq/libzmq.la b/external_libs/zmq/libzmq.la index 2e5f984d..2e5f984d 100755..100644 --- a/external_libs/cpp/zmq/libzmq.la +++ b/external_libs/zmq/libzmq.la diff --git a/external_libs/cpp/zmq/libzmq.lai b/external_libs/zmq/libzmq.lai index 126d3d5e..126d3d5e 100755..100644 --- a/external_libs/cpp/zmq/libzmq.lai +++ b/external_libs/zmq/libzmq.lai diff --git a/external_libs/cpp/zmq/libzmq.so b/external_libs/zmq/libzmq.so Binary files differindex 16980c27..16980c27 100755..100644 --- a/external_libs/cpp/zmq/libzmq.so +++ b/external_libs/zmq/libzmq.so diff --git a/external_libs/cpp/zmq/libzmq.so.3 b/external_libs/zmq/libzmq.so.3 Binary files differindex 16980c27..16980c27 100755..100644 --- a/external_libs/cpp/zmq/libzmq.so.3 +++ b/external_libs/zmq/libzmq.so.3 diff --git a/external_libs/cpp/zmq/libzmq.so.3.1.0 b/external_libs/zmq/libzmq.so.3.1.0 Binary files differindex 16980c27..16980c27 100755..100644 --- a/external_libs/cpp/zmq/libzmq.so.3.1.0 +++ b/external_libs/zmq/libzmq.so.3.1.0 diff --git a/linux/ws_main.py b/linux/ws_main.py index 1f096788..505853b6 100755 --- a/linux/ws_main.py +++ b/linux/ws_main.py @@ -159,7 +159,7 @@ rpc_server_mock_src = SrcGroup(dir='src/rpc-server/src', ]) # JSON package -json_src = SrcGroup(dir='external_libs/cpp/json', +json_src = SrcGroup(dir='external_libs/json', src_list=[ 'jsoncpp.cpp' ]) @@ -170,7 +170,7 @@ rpc_server_mock = SrcGroups([cmn_src, json_src ]) -yaml_src = SrcGroup(dir='external_libs/cpp/yaml-cpp/src/', +yaml_src = SrcGroup(dir='external_libs/yaml-cpp/src/', src_list=[ 'aliasmanager.cpp', 'binary.cpp', @@ -223,9 +223,9 @@ cxxflags_base =['-DWIN_UCODE_SIM', includes_path =''' ../src/pal/linux/ ../src/ ../src/rpc-server/include - ../external_libs/cpp/json/ - ../external_libs/cpp/zmq/include/ - ../external_libs/cpp/yaml-cpp/include/ + ../external_libs/json/ + ../external_libs/zmq/include/ + ../external_libs/yaml-cpp/include/ '''; @@ -357,7 +357,7 @@ build_types = [ def build_prog (bld, build_obj): - zmq_lib_path='external_libs/cpp/zmq/' + zmq_lib_path='external_libs/zmq/' bld.read_shlib( name='zmq' , paths=[top + zmq_lib_path] ) bld.program(features='cxx cxxprogram', diff --git a/linux_dpdk/ws_main.py b/linux_dpdk/ws_main.py index 817950f3..92364d75 100755 --- a/linux_dpdk/ws_main.py +++ b/linux_dpdk/ws_main.py @@ -131,7 +131,7 @@ net_src = SrcGroup(dir='src/common/Network/Packet', 'VLANHeader.cpp']); # JSON package -json_src = SrcGroup(dir='external_libs/cpp/json', +json_src = SrcGroup(dir='external_libs/json', src_list=[ 'jsoncpp.cpp' ]) @@ -149,12 +149,12 @@ rpc_server_src = SrcGroup(dir='src/rpc-server/src', ]) # JSON package -json_src = SrcGroup(dir='external_libs/cpp/json', +json_src = SrcGroup(dir='external_libs/json', src_list=[ 'jsoncpp.cpp' ]) -yaml_src = SrcGroup(dir='external_libs/cpp/yaml-cpp/src/', +yaml_src = SrcGroup(dir='external_libs/yaml-cpp/src/', src_list=[ 'aliasmanager.cpp', 'binary.cpp', @@ -401,9 +401,9 @@ includes_path =''' ../src/pal/linux_dpdk/ ../src/rpc-server/include - ../external_libs/cpp/yaml-cpp/include/ - ../external_libs/cpp/zmq/include/ - ../external_libs/cpp/json/ + ../external_libs/yaml-cpp/include/ + ../external_libs/zmq/include/ + ../external_libs/json/ ../src/dpdk_lib18/librte_eal/linuxapp/eal/include/ ../src/dpdk_lib18/librte_eal/common/include/ @@ -593,7 +593,7 @@ build_types = [ def build_prog (bld, build_obj): - zmq_lib_path='external_libs/cpp/zmq/' + zmq_lib_path='external_libs/zmq/' bld.read_shlib( name='zmq' , paths=[top+zmq_lib_path] ) #rte_libs =[ @@ -750,7 +750,7 @@ files_list=[ 'trex_daemon_server' ]; -files_dir=['cap2','avl','cfg','ko','automation','python-lib'] +files_dir=['cap2','avl','cfg','ko','automation', 'external_libs', 'python-lib'] class Env(object): diff --git a/scripts/automation/trex_control_plane/client/outer_packages.py b/scripts/automation/trex_control_plane/client/outer_packages.py index 886aef93..5facad20 100755 --- a/scripts/automation/trex_control_plane/client/outer_packages.py +++ b/scripts/automation/trex_control_plane/client/outer_packages.py @@ -6,8 +6,7 @@ import os CURRENT_PATH = os.path.dirname(os.path.realpath(__file__))
ROOT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir)) # path to trex_control_plane directory
-PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir,
- os.pardir, 'external_libs', 'python'))
+PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir, 'external_libs'))
CLIENT_MODULES = ['enum34-1.0.4',
'jsonrpclib-pelix-0.2.5',
diff --git a/scripts/automation/trex_control_plane/client_utils/outer_packages.py b/scripts/automation/trex_control_plane/client_utils/outer_packages.py index 53cce991..c489fd3d 100644 --- a/scripts/automation/trex_control_plane/client_utils/outer_packages.py +++ b/scripts/automation/trex_control_plane/client_utils/outer_packages.py @@ -6,8 +6,7 @@ import os CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) ROOT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir)) # path to trex_control_plane directory -PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir, - os.pardir, 'external_libs', 'python')) +PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir, 'external_libs')) CLIENT_UTILS_MODULES = ['zmq'] diff --git a/scripts/automation/trex_control_plane/server/outer_packages.py b/scripts/automation/trex_control_plane/server/outer_packages.py index f07ed59a..976e478d 100755 --- a/scripts/automation/trex_control_plane/server/outer_packages.py +++ b/scripts/automation/trex_control_plane/server/outer_packages.py @@ -6,8 +6,7 @@ import os CURRENT_PATH = os.path.dirname(os.path.realpath(__file__)) ROOT_PATH = os.path.abspath(os.path.join(CURRENT_PATH, os.pardir)) # path to trex_control_plane directory -PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir, - os.pardir, 'external_libs', 'python')) +PATH_TO_PYTHON_LIB = os.path.abspath(os.path.join(ROOT_PATH, os.pardir, os.pardir, 'external_libs')) SERVER_MODULES = ['enum34-1.0.4', 'jsonrpclib-pelix-0.2.5', diff --git a/scripts/external_libs/PyYAML-3.01/LICENSE b/scripts/external_libs/PyYAML-3.01/LICENSE new file mode 100644 index 00000000..050ced23 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2006 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scripts/external_libs/PyYAML-3.01/PKG-INFO b/scripts/external_libs/PyYAML-3.01/PKG-INFO new file mode 100644 index 00000000..6ec73b1f --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/PKG-INFO @@ -0,0 +1,28 @@ +Metadata-Version: 1.0 +Name: PyYAML +Version: 3.01 +Summary: YAML parser and emitter for Python +Home-page: http://pyyaml.org/wiki/PyYAML +Author: Kirill Simonov +Author-email: xi@resolvent.net +License: MIT +Download-URL: http://pyyaml.org/download/pyyaml/PyYAML-3.01.tar.gz +Description: YAML is a data serialization format designed for human readability and + interaction with scripting languages. PyYAML is a YAML parser and + emitter for Python. + + PyYAML features a complete YAML 1.1 parser, Unicode support, pickle + support, capable extension API, and sensible error messages. PyYAML + supports standard YAML tags and provides Python-specific tags that allow + to represent an arbitrary Python object. + + PyYAML is applicable for a broad range of tasks from complex + configuration files to object serialization and persistance. +Platform: Any +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing :: Markup diff --git a/scripts/external_libs/PyYAML-3.01/README b/scripts/external_libs/PyYAML-3.01/README new file mode 100644 index 00000000..8a6dec77 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/README @@ -0,0 +1,18 @@ +PyYAML 3000 - The next generation YAML parser and emitter for Python. + +To install, type 'python setup.py install'. + +For more information, check the PyYAML homepage: +'http://pyyaml.org/wiki/PyYAML'. + +Documentation (rough and incomplete though): +'http://pyyaml.org/wiki/PyYAMLDocumentation'. + +Post your questions and opinions to the YAML-Core mailing list: +'http://lists.sourceforge.net/lists/listinfo/yaml-core'. + +Submit bug reports and feature requests to the PyYAML bug tracker: +'http://pyyaml.org/newticket?component=pyyaml'. + +PyYAML 3000 is written by Kirill Simonov <xi@resolvent.net>. It is released +under the MIT license. See the file LICENSE for more details. diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/__init__.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/__init__.py new file mode 100644 index 00000000..c30973a3 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/__init__.py @@ -0,0 +1,284 @@ + +from error import * + +from tokens import * +from events import * +from nodes import * + +from loader import * +from dumper import * + +def scan(stream, Loader=Loader): + """ + Scan a YAML stream and produce scanning tokens. + """ + loader = Loader(stream) + while loader.check_token(): + yield loader.get_token() + +def parse(stream, Loader=Loader): + """ + Parse a YAML stream and produce parsing events. + """ + loader = Loader(stream) + while loader.check_event(): + yield loader.get_event() + +def compose(stream, Loader=Loader): + """ + Parse the first YAML document in a stream + and produce the corresponding representation tree. + """ + loader = Loader(stream) + if loader.check_node(): + return loader.get_node() + +def compose_all(stream, Loader=Loader): + """ + Parse all YAML documents in a stream + and produce corresponsing representation trees. + """ + loader = Loader(stream) + while loader.check_node(): + yield loader.get_node() + +def load_all(stream, Loader=Loader): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + """ + loader = Loader(stream) + while loader.check_data(): + yield loader.get_data() + +def load(stream, Loader=Loader): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + """ + loader = Loader(stream) + if loader.check_data(): + return loader.get_data() + +def safe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + Resolve only basic YAML tags. + """ + return load_all(stream, SafeLoader) + +def safe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + Resolve only basic YAML tags. + """ + return load(stream, SafeLoader) + +def emit(events, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + """ + Emit YAML parsing events into a stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + stream = StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + for event in events: + dumper.emit(event) + if getvalue: + return getvalue() + +def serialize_all(nodes, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding='utf-8', explicit_start=None, explicit_end=None, + version=None, tags=None): + """ + Serialize a sequence of representation trees into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + stream = StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end) + dumper.open() + for node in nodes: + dumper.serialize(node) + dumper.close() + if getvalue: + return getvalue() + +def serialize(node, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a representation tree into a YAML stream. + If stream is None, return the produced string instead. + """ + return serialize_all([node], stream, Dumper=Dumper, **kwds) + +def dump_all(documents, stream=None, Dumper=Dumper, + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding='utf-8', explicit_start=None, explicit_end=None, + version=None, tags=None): + """ + Serialize a sequence of Python objects into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + stream = StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end) + dumper.open() + for data in documents: + dumper.represent(data) + dumper.close() + if getvalue: + return getvalue() + +def dump(data, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a Python object into a YAML stream. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=Dumper, **kwds) + +def safe_dump_all(documents, stream=None, **kwds): + """ + Serialize a sequence of Python objects into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all(documents, stream, Dumper=SafeDumper, **kwds) + +def safe_dump(data, stream=None, **kwds): + """ + Serialize a Python object into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=SafeDumper, **kwds) + +def add_implicit_resolver(tag, regexp, first=None, + Loader=Loader, Dumper=Dumper): + """ + Add an implicit scalar detector. + If an implicit scalar value matches the given regexp, + the corresponding tag is assigned to the scalar. + first is a sequence of possible initial characters or None. + """ + Loader.add_implicit_resolver(tag, regexp, first) + Dumper.add_implicit_resolver(tag, regexp, first) + +def add_path_resolver(tag, path, kind=None, Loader=Loader, Dumper=Dumper): + """ + Add a path based resolver for the given tag. + A path is a list of keys that forms a path + to a node in the representation tree. + Keys can be string values, integers, or None. + """ + Loader.add_path_resolver(tag, path, kind) + Dumper.add_path_resolver(tag, path, kind) + +def add_constructor(tag, constructor, Loader=Loader): + """ + Add a constructor for the given tag. + Constructor is a function that accepts a Loader instance + and a node object and produces the corresponding Python object. + """ + Loader.add_constructor(tag, constructor) + +def add_multi_constructor(tag_prefix, multi_constructor, Loader=Loader): + """ + Add a multi-constructor for the given tag prefix. + Multi-constructor is called for a node if its tag starts with tag_prefix. + Multi-constructor accepts a Loader instance, a tag suffix, + and a node object and produces the corresponding Python object. + """ + Loader.add_multi_constructor(tag_prefix, multi_constructor) + +def add_representer(data_type, representer, Dumper=Dumper): + """ + Add a representer for the given type. + Representer is a function accepting a Dumper instance + and an instance of the given data type + and producing the corresponding representation node. + """ + Dumper.add_representer(data_type, representer) + +def add_multi_representer(data_type, multi_representer, Dumper=Dumper): + """ + Add a representer for the given type. + Multi-representer is a function accepting a Dumper instance + and an instance of the given data type or subtype + and producing the corresponding representation node. + """ + Dumper.add_multi_representer(data_type, multi_representer) + +class YAMLObjectMetaclass(type): + """ + The metaclass for YAMLObject. + """ + def __init__(cls, name, bases, kwds): + super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds) + if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None: + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + cls.yaml_dumper.add_representer(cls, cls.to_yaml) + +class YAMLObject(object): + """ + An object that can dump itself to a YAML stream + and load itself from a YAML stream. + """ + + __metaclass__ = YAMLObjectMetaclass + + yaml_loader = Loader + yaml_dumper = Dumper + + yaml_tag = None + yaml_flow_style = None + + def from_yaml(cls, loader, node): + """ + Convert a representation node to a Python object. + """ + return loader.construct_yaml_object(node, cls) + from_yaml = classmethod(from_yaml) + + def to_yaml(cls, dumper, data): + """ + Convert a Python object to a representation node. + """ + return dumper.represent_yaml_object(cls.yaml_tag, data, cls, + flow_style=cls.yaml_flow_style) + to_yaml = classmethod(to_yaml) + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/composer.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/composer.py new file mode 100644 index 00000000..d256b054 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/composer.py @@ -0,0 +1,123 @@ + +__all__ = ['Composer', 'ComposerError'] + +from error import MarkedYAMLError +from events import * +from nodes import * + +class ComposerError(MarkedYAMLError): + pass + +class Composer: + + def __init__(self): + self.anchors = {} + + def check_node(self): + # If there are more documents available? + return not self.check_event(StreamEndEvent) + + def get_node(self): + # Get the root node of the next document. + if not self.check_event(StreamEndEvent): + return self.compose_document() + + def __iter__(self): + # Iterator protocol. + while not self.check_event(StreamEndEvent): + yield self.compose_document() + + def compose_document(self): + + # Drop the STREAM-START event. + if self.check_event(StreamStartEvent): + self.get_event() + + # Drop the DOCUMENT-START event. + self.get_event() + + # Compose the root node. + node = self.compose_node(None, None) + + # Drop the DOCUMENT-END event. + self.get_event() + + self.complete_anchors = {} + return node + + def compose_node(self, parent, index): + if self.check_event(AliasEvent): + event = self.get_event() + anchor = event.anchor + if anchor not in self.anchors: + raise ComposerError(None, None, "found undefined alias %r" + % anchor.encode('utf-8'), event.start_mark) + return self.anchors[anchor] + event = self.peek_event() + anchor = event.anchor + if anchor is not None: + if anchor in self.anchors: + raise ComposerError("found duplicate anchor %r; first occurence" + % anchor.encode('utf-8'), self.anchors[anchor].start_mark, + "second occurence", event.start_mark) + self.descend_resolver(parent, index) + if self.check_event(ScalarEvent): + node = self.compose_scalar_node(anchor) + elif self.check_event(SequenceStartEvent): + node = self.compose_sequence_node(anchor) + elif self.check_event(MappingStartEvent): + node = self.compose_mapping_node(anchor) + self.ascend_resolver() + return node + + def compose_scalar_node(self, anchor): + event = self.get_event() + tag = event.tag + if tag is None or tag == u'!': + tag = self.resolve(ScalarNode, event.value, event.implicit) + node = ScalarNode(tag, event.value, + event.start_mark, event.end_mark, style=event.style) + if anchor is not None: + self.anchors[anchor] = node + return node + + def compose_sequence_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == u'!': + tag = self.resolve(SequenceNode, None, start_event.implicit) + node = SequenceNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + index = 0 + while not self.check_event(SequenceEndEvent): + node.value.append(self.compose_node(node, index)) + index += 1 + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + + def compose_mapping_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == u'!': + tag = self.resolve(MappingNode, None, start_event.implicit) + node = MappingNode(tag, {}, + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + while not self.check_event(MappingEndEvent): + key_event = self.peek_event() + item_key = self.compose_node(node, None) + if item_key in node.value: + raise ComposerError("while composing a mapping", start_event.start_mark, + "found duplicate key", key_event.start_mark) + item_value = self.compose_node(node, item_key) + node.value[item_key] = item_value + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/constructor.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/constructor.py new file mode 100644 index 00000000..57ad53d1 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/constructor.py @@ -0,0 +1,638 @@ + +__all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor', + 'ConstructorError'] + +from error import * +from nodes import * +from composer import * + +try: + import datetime + datetime_available = True +except ImportError: + datetime_available = False + +try: + set +except NameError: + from sets import Set as set + +import binascii, re, sys + +class ConstructorError(MarkedYAMLError): + pass + +class BaseConstructor(Composer): + + yaml_constructors = {} + yaml_multi_constructors = {} + + def __init__(self): + self.constructed_objects = {} + self.recursive_objects = {} + + def check_data(self): + # If there are more documents available? + return self.check_node() + + def get_data(self): + # Construct and return the next document. + if self.check_node(): + return self.construct_document(self.get_node()) + + def __iter__(self): + # Iterator protocol. + while self.check_node(): + yield self.construct_document(self.get_node()) + + def construct_document(self, node): + data = self.construct_object(node) + self.constructed_objects = {} + self.recursive_objects = {} + return data + + def construct_object(self, node): + if node in self.constructed_objects: + return self.constructed_objects[node] + if node in self.recursive_objects: + raise ConstructorError(None, None, + "found recursive node", node.start_mark) + self.recursive_objects[node] = None + constructor = None + if node.tag in self.yaml_constructors: + constructor = lambda node: self.yaml_constructors[node.tag](self, node) + else: + for tag_prefix in self.yaml_multi_constructors: + if node.tag.startswith(tag_prefix): + tag_suffix = node.tag[len(tag_prefix):] + constructor = lambda node: \ + self.yaml_multi_constructors[tag_prefix](self, tag_suffix, node) + break + else: + if None in self.yaml_multi_constructors: + constructor = lambda node: \ + self.yaml_multi_constructors[None](self, node.tag, node) + elif None in self.yaml_constructors: + constructor = lambda node: \ + self.yaml_constructors[None](self, node) + elif isinstance(node, ScalarNode): + constructor = self.construct_scalar + elif isinstance(node, SequenceNode): + constructor = self.construct_sequence + elif isinstance(node, MappingNode): + constructor = self.construct_mapping + else: + print node.tag + data = constructor(node) + self.constructed_objects[node] = data + del self.recursive_objects[node] + return data + + def construct_scalar(self, node): + if not isinstance(node, ScalarNode): + if isinstance(node, MappingNode): + for key_node in node.value: + if key_node.tag == u'tag:yaml.org,2002:value': + return self.construct_scalar(node.value[key_node]) + raise ConstructorError(None, None, + "expected a scalar node, but found %s" % node.id, + node.start_mark) + return node.value + + def construct_sequence(self, node): + if not isinstance(node, SequenceNode): + raise ConstructorError(None, None, + "expected a sequence node, but found %s" % node.id, + node.start_mark) + return [self.construct_object(child) for child in node.value] + + def construct_mapping(self, node): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + mapping = {} + merge = None + for key_node in node.value: + if key_node.tag == u'tag:yaml.org,2002:merge': + if merge is not None: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found duplicate merge key", key_node.start_mark) + value_node = node.value[key_node] + if isinstance(value_node, MappingNode): + merge = [self.construct_mapping(value_node)] + elif isinstance(value_node, SequenceNode): + merge = [] + for subnode in value_node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing a mapping", + node.start_mark, + "expected a mapping for merging, but found %s" + % subnode.id, subnode.start_mark) + merge.append(self.construct_mapping(subnode)) + merge.reverse() + else: + raise ConstructorError("while constructing a mapping", node.start_mark, + "expected a mapping or list of mappings for merging, but found %s" + % value_node.id, value_node.start_mark) + elif key_node.tag == u'tag:yaml.org,2002:value': + if '=' in mapping: + raise ConstructorError("while construction a mapping", node.start_mark, + "found duplicate value key", key_node.start_mark) + value = self.construct_object(node.value[key_node]) + mapping['='] = value + else: + key = self.construct_object(key_node) + try: + duplicate_key = key in mapping + except TypeError, exc: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unacceptable key (%s)" % exc, key_node.start_mark) + if duplicate_key: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found duplicate key", key_node.start_mark) + value = self.construct_object(node.value[key_node]) + mapping[key] = value + if merge is not None: + merge.append(mapping) + mapping = {} + for submapping in merge: + mapping.update(submapping) + return mapping + + def construct_pairs(self, node): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + pairs = [] + for key_node in node.value: + key = self.construct_object(key_node) + value = self.construct_object(node.value[key_node]) + pairs.append((key, value)) + return pairs + + def add_constructor(cls, tag, constructor): + if not 'yaml_constructors' in cls.__dict__: + cls.yaml_constructors = cls.yaml_constructors.copy() + cls.yaml_constructors[tag] = constructor + add_constructor = classmethod(add_constructor) + + def add_multi_constructor(cls, tag_prefix, multi_constructor): + if not 'yaml_multi_constructors' in cls.__dict__: + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() + cls.yaml_multi_constructors[tag_prefix] = multi_constructor + add_multi_constructor = classmethod(add_multi_constructor) + +class SafeConstructor(BaseConstructor): + + def construct_yaml_null(self, node): + self.construct_scalar(node) + return None + + bool_values = { + u'yes': True, + u'no': False, + u'true': True, + u'false': False, + u'on': True, + u'off': False, + } + + def construct_yaml_bool(self, node): + value = self.construct_scalar(node) + return self.bool_values[value.lower()] + + def construct_yaml_int(self, node): + value = str(self.construct_scalar(node)) + value = value.replace('_', '') + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '0': + return 0 + elif value.startswith('0b'): + return sign*int(value[2:], 2) + elif value.startswith('0x'): + return sign*int(value[2:], 16) + elif value[0] == '0': + return sign*int(value, 8) + elif ':' in value: + digits = [int(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*int(value) + + inf_value = 1e300000 + nan_value = inf_value/inf_value + + def construct_yaml_float(self, node): + value = str(self.construct_scalar(node)) + value = value.replace('_', '') + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value.lower() == '.inf': + return sign*self.inf_value + elif value.lower() == '.nan': + return self.nan_value + elif ':' in value: + digits = [float(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0.0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return float(value) + + def construct_yaml_binary(self, node): + value = self.construct_scalar(node) + try: + return str(value).decode('base64') + except (binascii.Error, UnicodeEncodeError), exc: + raise ConstructorError(None, None, + "failed to decode base64 data: %s" % exc, node.start_mark) + + timestamp_regexp = re.compile( + ur'''^(?P<year>[0-9][0-9][0-9][0-9]) + -(?P<month>[0-9][0-9]?) + -(?P<day>[0-9][0-9]?) + (?:(?:[Tt]|[ \t]+) + (?P<hour>[0-9][0-9]?) + :(?P<minute>[0-9][0-9]) + :(?P<second>[0-9][0-9]) + (?:\.(?P<fraction>[0-9]*))? + (?:[ \t]*(?:Z|(?P<tz_hour>[-+][0-9][0-9]?) + (?::(?P<tz_minute>[0-9][0-9])?)?))?)?$''', re.X) + + def construct_yaml_timestamp(self, node): + value = self.construct_scalar(node) + match = self.timestamp_regexp.match(node.value) + values = match.groupdict() + for key in values: + if values[key]: + values[key] = int(values[key]) + else: + values[key] = 0 + fraction = values['fraction'] + if fraction: + while 10*fraction < 1000000: + fraction *= 10 + values['fraction'] = fraction + stamp = datetime.datetime(values['year'], values['month'], values['day'], + values['hour'], values['minute'], values['second'], values['fraction']) + diff = datetime.timedelta(hours=values['tz_hour'], minutes=values['tz_minute']) + return stamp-diff + + def construct_yaml_omap(self, node): + # Note: we do not check for duplicate keys, because it's too + # CPU-expensive. + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + omap = [] + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node = subnode.value.keys()[0] + key = self.construct_object(key_node) + value = self.construct_object(subnode.value[key_node]) + omap.append((key, value)) + return omap + + def construct_yaml_pairs(self, node): + # Note: the same code as `construct_yaml_omap`. + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + pairs = [] + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node = subnode.value.keys()[0] + key = self.construct_object(key_node) + value = self.construct_object(subnode.value[key_node]) + pairs.append((key, value)) + return pairs + + def construct_yaml_set(self, node): + value = self.construct_mapping(node) + return set(value) + + def construct_yaml_str(self, node): + value = self.construct_scalar(node) + try: + return str(value) + except UnicodeEncodeError: + return value + + def construct_yaml_seq(self, node): + return self.construct_sequence(node) + + def construct_yaml_map(self, node): + return self.construct_mapping(node) + + def construct_yaml_object(self, node, cls): + state = self.construct_mapping(node) + data = cls.__new__(cls) + if hasattr(data, '__setstate__'): + data.__setstate__(state) + else: + data.__dict__.update(state) + return data + + def construct_undefined(self, node): + raise ConstructorError(None, None, + "could not determine a constructor for the tag %r" % node.tag.encode('utf-8'), + node.start_mark) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:null', + SafeConstructor.construct_yaml_null) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:bool', + SafeConstructor.construct_yaml_bool) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:int', + SafeConstructor.construct_yaml_int) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:float', + SafeConstructor.construct_yaml_float) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:binary', + SafeConstructor.construct_yaml_binary) + +if datetime_available: + SafeConstructor.add_constructor( + u'tag:yaml.org,2002:timestamp', + SafeConstructor.construct_yaml_timestamp) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:omap', + SafeConstructor.construct_yaml_omap) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:pairs', + SafeConstructor.construct_yaml_pairs) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:set', + SafeConstructor.construct_yaml_set) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:str', + SafeConstructor.construct_yaml_str) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:seq', + SafeConstructor.construct_yaml_seq) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:map', + SafeConstructor.construct_yaml_map) + +SafeConstructor.add_constructor(None, + SafeConstructor.construct_undefined) + +class Constructor(SafeConstructor): + + def construct_python_str(self, node): + return self.construct_scalar(node).encode('utf-8') + + def construct_python_unicode(self, node): + return self.construct_scalar(node) + + def construct_python_long(self, node): + return long(self.construct_yaml_int(node)) + + def construct_python_complex(self, node): + return complex(self.construct_scalar(node)) + + def construct_python_tuple(self, node): + return tuple(self.construct_yaml_seq(node)) + + def find_python_module(self, name, mark): + if not name: + raise ConstructorError("while constructing a Python module", mark, + "expected non-empty name appended to the tag", mark) + try: + __import__(name) + except ImportError, exc: + raise ConstructorError("while constructing a Python module", mark, + "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark) + return sys.modules[name] + + def find_python_name(self, name, mark): + if not name: + raise ConstructorError("while constructing a Python object", mark, + "expected non-empty name appended to the tag", mark) + if u'.' in name: + # Python 2.4 only + #module_name, object_name = name.rsplit('.', 1) + items = name.split('.') + object_name = items.pop() + module_name = '.'.join(items) + else: + module_name = '__builtin__' + object_name = name + try: + __import__(module_name) + except ImportError, exc: + raise ConstructorError("while constructing a Python object", mark, + "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark) + module = sys.modules[module_name] + if not hasattr(module, object_name): + raise ConstructorError("while constructing a Python object", mark, + "cannot find %r in the module %r" % (object_name.encode('utf-8'), + module.__name__), mark) + return getattr(module, object_name) + + def construct_python_name(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python name", node.start_mark, + "expected the empty value, but found %r" % value.encode('utf-8'), + node.start_mark) + return self.find_python_name(suffix, node.start_mark) + + def construct_python_module(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python module", node.start_mark, + "expected the empty value, but found %r" % value.encode('utf-8'), + node.start_mark) + return self.find_python_module(suffix, node.start_mark) + + class classobj: pass + + def make_python_instance(self, suffix, node, + args=None, kwds=None, newobj=False): + if not args: + args = [] + if not kwds: + kwds = {} + cls = self.find_python_name(suffix, node.start_mark) + if newobj and isinstance(cls, type(self.classobj)) \ + and not args and not kwds: + instance = self.classobj() + instance.__class__ = cls + return instance + elif newobj and isinstance(cls, type): + return cls.__new__(cls, *args, **kwds) + else: + return cls(*args, **kwds) + + def set_python_instance_state(self, instance, state): + if hasattr(instance, '__setstate__'): + instance.__setstate__(state) + else: + slotstate = {} + if isinstance(state, tuple) and len(state) == 2: + state, slotstate = state + if hasattr(instance, '__dict__'): + instance.__dict__.update(state) + elif state: + slotstate.update(state) + for key, value in slotstate.items(): + setattr(object, key, value) + + def construct_python_object(self, suffix, node): + # Format: + # !!python/object:module.name { ... state ... } + instance = self.make_python_instance(suffix, node, newobj=True) + state = self.construct_mapping(node) + self.set_python_instance_state(instance, state) + return instance + + def construct_python_object_apply(self, suffix, node, newobj=False): + # Format: + # !!python/object/apply # (or !!python/object/new) + # args: [ ... arguments ... ] + # kwds: { ... keywords ... } + # state: ... state ... + # listitems: [ ... listitems ... ] + # dictitems: { ... dictitems ... } + # or short format: + # !!python/object/apply [ ... arguments ... ] + # The difference between !!python/object/apply and !!python/object/new + # is how an object is created, check make_python_instance for details. + if isinstance(node, SequenceNode): + args = self.construct_sequence(node) + kwds = {} + state = {} + listitems = [] + dictitems = {} + else: + value = self.construct_mapping(node) + args = value.get('args', []) + kwds = value.get('kwds', {}) + state = value.get('state', {}) + listitems = value.get('listitems', []) + dictitems = value.get('dictitems', {}) + instance = self.make_python_instance(suffix, node, args, kwds, newobj) + if state: + self.set_python_instance_state(instance, state) + if listitems: + instance.extend(listitems) + if dictitems: + for key in dictitems: + instance[key] = dictitems[key] + return instance + + def construct_python_object_new(self, suffix, node): + return self.construct_python_object_apply(suffix, node, newobj=True) + + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/none', + Constructor.construct_yaml_null) + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/bool', + Constructor.construct_yaml_bool) + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/str', + Constructor.construct_python_str) + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/unicode', + Constructor.construct_python_unicode) + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/int', + Constructor.construct_yaml_int) + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/long', + Constructor.construct_python_long) + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/float', + Constructor.construct_yaml_float) + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/complex', + Constructor.construct_python_complex) + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/list', + Constructor.construct_yaml_seq) + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/tuple', + Constructor.construct_python_tuple) + +Constructor.add_constructor( + u'tag:yaml.org,2002:python/dict', + Constructor.construct_yaml_map) + +Constructor.add_multi_constructor( + u'tag:yaml.org,2002:python/name:', + Constructor.construct_python_name) + +Constructor.add_multi_constructor( + u'tag:yaml.org,2002:python/module:', + Constructor.construct_python_module) + +Constructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object:', + Constructor.construct_python_object) + +Constructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object/apply:', + Constructor.construct_python_object_apply) + +Constructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object/new:', + Constructor.construct_python_object_new) + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/dumper.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/dumper.py new file mode 100644 index 00000000..355c1e2f --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/dumper.py @@ -0,0 +1,62 @@ + +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper'] + +from emitter import * +from serializer import * +from representer import * +from resolver import * + +class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_uncode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style) + Resolver.__init__(self) + +class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style) + Resolver.__init__(self) + +class Dumper(Emitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=None, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style) + Resolver.__init__(self) + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/emitter.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/emitter.py new file mode 100644 index 00000000..a34c4526 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/emitter.py @@ -0,0 +1,1162 @@ + +# Emitter expects events obeying the following grammar: +# stream ::= STREAM-START document* STREAM-END +# document ::= DOCUMENT-START node DOCUMENT-END +# node ::= SCALAR | sequence | mapping +# sequence ::= SEQUENCE-START node* SEQUENCE-END +# mapping ::= MAPPING-START (node node)* MAPPING-END + +__all__ = ['Emitter', 'EmitterError'] + +from error import YAMLError +from events import * + +import re + +class EmitterError(YAMLError): + pass + +class ScalarAnalysis: + def __init__(self, scalar, empty, multiline, + allow_flow_plain, allow_block_plain, + allow_single_quoted, allow_double_quoted, + allow_block): + self.scalar = scalar + self.empty = empty + self.multiline = multiline + self.allow_flow_plain = allow_flow_plain + self.allow_block_plain = allow_block_plain + self.allow_single_quoted = allow_single_quoted + self.allow_double_quoted = allow_double_quoted + self.allow_block = allow_block + +class Emitter: + + DEFAULT_TAG_PREFIXES = { + u'!' : u'!', + u'tag:yaml.org,2002:' : u'!!', + } + + def __init__(self, stream, canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + + # The stream should have the methods `write` and possibly `flush`. + self.stream = stream + + # Encoding can be overriden by STREAM-START. + self.encoding = None + + # Emitter is a state machine with a stack of states to handle nested + # structures. + self.states = [] + self.state = self.expect_stream_start + + # Current event and the event queue. + self.events = [] + self.event = None + + # The current indentation level and the stack of previous indents. + self.indents = [] + self.indent = None + + # Flow level. + self.flow_level = 0 + + # Contexts. + self.root_context = False + self.sequence_context = False + self.mapping_context = False + self.simple_key_context = False + + # Characteristics of the last emitted character: + # - current position. + # - is it a whitespace? + # - is it an indention character + # (indentation space, '-', '?', or ':')? + self.line = 0 + self.column = 0 + self.whitespace = True + self.indention = True + + # Formatting details. + self.canonical = canonical + self.allow_unicode = allow_unicode + self.best_indent = 2 + if indent and 1 < indent < 10: + self.best_indent = indent + self.best_width = 80 + if width and width > self.best_indent*2: + self.best_width = width + self.best_line_break = u'\n' + if line_break in [u'\r', u'\n', u'\r\n']: + self.best_line_break = line_break + + # Tag prefixes. + self.tag_prefixes = None + + # Prepared anchor and tag. + self.prepared_anchor = None + self.prepared_tag = None + + # Scalar analysis and style. + self.analysis = None + self.style = None + + def emit(self, event): + self.events.append(event) + while not self.need_more_events(): + self.event = self.events.pop(0) + self.state() + self.event = None + + # In some cases, we wait for a few next events before emitting. + + def need_more_events(self): + if not self.events: + return True + event = self.events[0] + if isinstance(event, DocumentStartEvent): + return self.need_events(1) + elif isinstance(event, SequenceStartEvent): + return self.need_events(2) + elif isinstance(event, MappingStartEvent): + return self.need_events(3) + else: + return False + + def need_events(self, count): + level = 0 + for event in self.events[1:]: + if isinstance(event, (DocumentStartEvent, CollectionStartEvent)): + level += 1 + elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)): + level -= 1 + elif isinstance(event, StreamEndEvent): + level = -1 + if level < 0: + return False + return (len(self.events) < count+1) + + def increase_indent(self, flow=False, indentless=False): + self.indents.append(self.indent) + if self.indent is None: + if flow: + self.indent = self.best_indent + else: + self.indent = 0 + elif not indentless: + self.indent += self.best_indent + + # States. + + # Stream handlers. + + def expect_stream_start(self): + if isinstance(self.event, StreamStartEvent): + if self.event.encoding: + self.encoding = self.event.encoding + self.write_stream_start() + self.state = self.expect_first_document_start + else: + raise EmitterError("expected StreamStartEvent, but got %s" + % self.event) + + def expect_nothing(self): + raise EmitterError("expected nothing, but got %s" % self.event) + + # Document handlers. + + def expect_first_document_start(self): + return self.expect_document_start(first=True) + + def expect_document_start(self, first=False): + if isinstance(self.event, DocumentStartEvent): + if self.event.version: + version_text = self.prepare_version(self.event.version) + self.write_version_directive(version_text) + self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy() + if self.event.tags: + handles = self.event.tags.keys() + handles.sort() + for handle in handles: + prefix = self.event.tags[handle] + self.tag_prefixes[prefix] = handle + handle_text = self.prepare_tag_handle(handle) + prefix_text = self.prepare_tag_prefix(prefix) + self.write_tag_directive(handle_text, prefix_text) + implicit = (first and not self.event.explicit and not self.canonical + and not self.event.version and not self.event.tags + and not self.check_empty_document()) + if not implicit: + self.write_indent() + self.write_indicator(u'---', True) + if self.canonical: + self.write_indent() + self.state = self.expect_document_root + elif isinstance(self.event, StreamEndEvent): + self.write_stream_end() + self.state = self.expect_nothing + else: + raise EmitterError("expected DocumentStartEvent, but got %s" + % self.event) + + def expect_document_end(self): + if isinstance(self.event, DocumentEndEvent): + self.write_indent() + if self.event.explicit: + self.write_indicator(u'...', True) + self.write_indent() + self.flush_stream() + self.state = self.expect_document_start + else: + raise EmitterError("expected DocumentEndEvent, but got %s" + % self.event) + + def expect_document_root(self): + self.states.append(self.expect_document_end) + self.expect_node(root=True) + + # Node handlers. + + def expect_node(self, root=False, sequence=False, mapping=False, + simple_key=False): + self.root_context = root + self.sequence_context = sequence + self.mapping_context = mapping + self.simple_key_context = simple_key + if isinstance(self.event, AliasEvent): + self.expect_alias() + elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)): + self.process_anchor(u'&') + self.process_tag() + if isinstance(self.event, ScalarEvent): + self.expect_scalar() + elif isinstance(self.event, SequenceStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_sequence(): + self.expect_flow_sequence() + else: + self.expect_block_sequence() + elif isinstance(self.event, MappingStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_mapping(): + self.expect_flow_mapping() + else: + self.expect_block_mapping() + else: + raise EmitterError("expected NodeEvent, but got %s" % self.event) + + def expect_alias(self): + if self.event.anchor is None: + raise EmitterError("anchor is not specified for alias") + self.process_anchor(u'*') + self.state = self.states.pop() + + def expect_scalar(self): + self.increase_indent(flow=True) + self.process_scalar() + self.indent = self.indents.pop() + self.state = self.states.pop() + + # Flow sequence handlers. + + def expect_flow_sequence(self): + self.write_indicator(u'[', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_sequence_item + + def expect_first_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator(u']', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + def expect_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(u',', False) + self.write_indent() + self.write_indicator(u']', False) + self.state = self.states.pop() + else: + self.write_indicator(u',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + # Flow mapping handlers. + + def expect_flow_mapping(self): + self.write_indicator(u'{', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_mapping_key + + def expect_first_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator(u'}', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator(u'?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(u',', False) + self.write_indent() + self.write_indicator(u'}', False) + self.state = self.states.pop() + else: + self.write_indicator(u',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator(u'?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_simple_value(self): + self.write_indicator(u':', False) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + def expect_flow_mapping_value(self): + if self.canonical or self.column > self.best_width: + self.write_indent() + self.write_indicator(u':', True) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + # Block sequence handlers. + + def expect_block_sequence(self): + indentless = (self.mapping_context and not self.indention) + self.increase_indent(flow=False, indentless=indentless) + self.state = self.expect_first_block_sequence_item + + def expect_first_block_sequence_item(self): + return self.expect_block_sequence_item(first=True) + + def expect_block_sequence_item(self, first=False): + if not first and isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + self.write_indicator(u'-', True, indention=True) + self.states.append(self.expect_block_sequence_item) + self.expect_node(sequence=True) + + # Block mapping handlers. + + def expect_block_mapping(self): + self.increase_indent(flow=False) + self.state = self.expect_first_block_mapping_key + + def expect_first_block_mapping_key(self): + return self.expect_block_mapping_key(first=True) + + def expect_block_mapping_key(self, first=False): + if not first and isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + if self.check_simple_key(): + self.states.append(self.expect_block_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator(u'?', True, indention=True) + self.states.append(self.expect_block_mapping_value) + self.expect_node(mapping=True) + + def expect_block_mapping_simple_value(self): + self.write_indicator(u':', False) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + def expect_block_mapping_value(self): + self.write_indent() + self.write_indicator(u':', True, indention=True) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + # Checkers. + + def check_empty_sequence(self): + return (isinstance(self.event, SequenceStartEvent) and self.events + and isinstance(self.events[0], SequenceEndEvent)) + + def check_empty_mapping(self): + return (isinstance(self.event, MappingStartEvent) and self.events + and isinstance(self.events[0], MappingEndEvent)) + + def check_empty_document(self): + if not isinstance(self.event, DocumentStartEvent) or not self.events: + return False + event = self.events[0] + return (isinstance(event, ScalarEvent) and event.anchor is None + and event.tag is None and event.implicit and event.value == u'') + + def check_simple_key(self): + length = 0 + if isinstance(self.event, NodeEvent) and self.event.anchor is not None: + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + length += len(self.prepared_anchor) + if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \ + and self.event.tag is not None: + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(self.event.tag) + length += len(self.prepared_tag) + if isinstance(self.event, ScalarEvent): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + length += len(self.analysis.scalar) + return (length < 128 and (isinstance(self.event, AliasEvent) + or (isinstance(self.event, ScalarEvent) + and not self.analysis.empty and not self.analysis.multiline) + or self.check_empty_sequence() or self.check_empty_mapping())) + + # Anchor, Tag, and Scalar processors. + + def process_anchor(self, indicator): + if self.event.anchor is None: + self.prepared_anchor = None + return + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + if self.prepared_anchor: + self.write_indicator(indicator+self.prepared_anchor, True) + self.prepared_anchor = None + + def process_tag(self): + tag = self.event.tag + if isinstance(self.event, ScalarEvent): + if self.style is None: + self.style = self.choose_scalar_style() + if ((not self.canonical or tag is None) and + ((self.style == '' and self.event.implicit[0]) + or (self.style != '' and self.event.implicit[1]))): + self.prepared_tag = None + return + if self.event.implicit[0] and tag is None: + tag = u'!' + self.prepared_tag = None + else: + if (not self.canonical or tag is None) and self.event.implicit: + self.prepared_tag = None + return + if tag is None: + raise EmitterError("tag is not specified") + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(tag) + if self.prepared_tag: + self.write_indicator(self.prepared_tag, True) + self.prepared_tag = None + + def choose_scalar_style(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.event.style == '"' or self.canonical: + return '"' + if not self.event.style and self.event.implicit[0]: + if (not (self.simple_key_context and + (self.analysis.empty or self.analysis.multiline)) + and (self.flow_level and self.analysis.allow_flow_plain + or (not self.flow_level and self.analysis.allow_block_plain))): + return '' + if self.event.style and self.event.style in '|>': + if not self.flow_level and self.analysis.allow_block: + return self.event.style + if not self.event.style or self.event.style == '\'': + if (self.analysis.allow_single_quoted and + not (self.simple_key_context and self.analysis.multiline)): + return '\'' + return '"' + + def process_scalar(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.style is None: + self.style = self.choose_scalar_style() + split = (not self.simple_key_context) + #if self.analysis.multiline and split \ + # and (not self.style or self.style in '\'\"'): + # self.write_indent() + if self.style == '"': + self.write_double_quoted(self.analysis.scalar, split) + elif self.style == '\'': + self.write_single_quoted(self.analysis.scalar, split) + elif self.style == '>': + self.write_folded(self.analysis.scalar) + elif self.style == '|': + self.write_literal(self.analysis.scalar) + else: + self.write_plain(self.analysis.scalar, split) + self.analysis = None + self.style = None + + # Analyzers. + + def prepare_version(self, version): + major, minor = version + if major != 1: + raise EmitterError("unsupported YAML version: %d.%d" % (major, minor)) + return u'%d.%d' % (major, minor) + + def prepare_tag_handle(self, handle): + if not handle: + raise EmitterError("tag handle must not be empty") + if handle[0] != u'!' or handle[-1] != u'!': + raise EmitterError("tag handle must start and end with '!': %r" + % (handle.encode('utf-8'))) + for ch in handle[1:-1]: + if not (u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \ + or ch in u'-_'): + raise EmitterError("invalid character %r in the tag handle: %r" + % (ch.encode('utf-8'), handle.encode('utf-8'))) + return handle + + def prepare_tag_prefix(self, prefix): + if not prefix: + raise EmitterError("tag prefix must not be empty") + chunks = [] + start = end = 0 + if prefix[0] == u'!': + end = 1 + while end < len(prefix): + ch = prefix[end] + if u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \ + or ch in u'-;/?!:@&=+$,_.~*\'()[]': + end += 1 + else: + if start < end: + chunks.append(prefix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append(u'%%%02X' % ord(ch)) + if start < end: + chunks.append(prefix[start:end]) + return u''.join(chunks) + + def prepare_tag(self, tag): + if not tag: + raise EmitterError("tag must not be empty") + if tag == u'!': + return tag + handle = None + suffix = tag + for prefix in self.tag_prefixes: + if tag.startswith(prefix) \ + and (prefix == u'!' or len(prefix) < len(tag)): + handle = self.tag_prefixes[prefix] + suffix = tag[len(prefix):] + chunks = [] + start = end = 0 + while end < len(suffix): + ch = suffix[end] + if u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \ + or ch in u'-;/?:@&=+$,_.~*\'()[]' \ + or (ch == u'!' and handle != u'!'): + end += 1 + else: + if start < end: + chunks.append(suffix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append(u'%%%02X' % ord(ch)) + if start < end: + chunks.append(suffix[start:end]) + suffix_text = u''.join(chunks) + if handle: + return u'%s%s' % (handle, suffix_text) + else: + return u'!<%s>' % suffix_text + + def prepare_anchor(self, anchor): + if not anchor: + raise EmitterError("anchor must not be empty") + for ch in anchor: + if not (u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \ + or ch in u'-_'): + raise EmitterError("invalid character %r in the anchor: %r" + % (ch.encode('utf-8'), anchor.encode('utf-8'))) + return anchor + + def analyze_scalar(self, scalar): + + # Empty scalar is a special case. + if not scalar: + return ScalarAnalysis(scalar=scalar, empty=True, multiline=False, + allow_flow_plain=False, allow_block_plain=True, + allow_single_quoted=True, allow_double_quoted=True, + allow_block=False) + + # Indicators and special characters. + block_indicators = False + flow_indicators = False + line_breaks = False + special_characters = False + + # Whitespaces. + inline_spaces = False # non-space space+ non-space + inline_breaks = False # non-space break+ non-space + leading_spaces = False # ^ space+ (non-space | $) + leading_breaks = False # ^ break+ (non-space | $) + trailing_spaces = False # (^ | non-space) space+ $ + trailing_breaks = False # (^ | non-space) break+ $ + inline_breaks_spaces = False # non-space break+ space+ non-space + mixed_breaks_spaces = False # anything else + + # Check document indicators. + if scalar.startswith(u'---') or scalar.startswith(u'...'): + block_indicators = True + flow_indicators = True + + # First character or preceded by a whitespace. + preceeded_by_space = True + + # Last character or followed by a whitespace. + followed_by_space = (len(scalar) == 1 or + scalar[1] in u'\0 \t\r\n\x85\u2028\u2029') + + # The current series of whitespaces contain plain spaces. + spaces = False + + # The current series of whitespaces contain line breaks. + breaks = False + + # The current series of whitespaces contain a space followed by a + # break. + mixed = False + + # The current series of whitespaces start at the beginning of the + # scalar. + leading = False + + index = 0 + while index < len(scalar): + ch = scalar[index] + + # Check for indicators. + + if index == 0: + # Leading indicators are special characters. + if ch in u'#,[]{}#&*!|>\'\"%@`': + flow_indicators = True + block_indicators = True + if ch in u'?:': + flow_indicators = True + if followed_by_space: + block_indicators = True + if ch == u'-' and followed_by_space: + flow_indicators = True + block_indicators = True + else: + # Some indicators cannot appear within a scalar as well. + if ch in u',?[]{}': + flow_indicators = True + if ch == u':': + flow_indicators = True + if followed_by_space: + block_indicators = True + if ch == u'#' and preceeded_by_space: + flow_indicators = True + block_indicators = True + + # Check for line breaks, special, and unicode characters. + + if ch in u'\n\x85\u2028\u2029': + line_breaks = True + if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'): + if (ch == u'\x85' or u'\xA0' <= ch <= u'\uD7FF' + or u'\uE000' <= ch <= u'\uFFFD') and ch != u'\uFEFF': + unicode_characters = True + if not self.allow_unicode: + special_characters = True + else: + special_characters = True + + # Spaces, line breaks, and how they are mixed. State machine. + + # Start or continue series of whitespaces. + if ch in u' \n\x85\u2028\u2029': + if spaces and breaks: + if ch != u' ': # break+ (space+ break+) => mixed + mixed = True + elif spaces: + if ch != u' ': # (space+ break+) => mixed + breaks = True + mixed = True + elif breaks: + if ch == u' ': # break+ space+ + spaces = True + else: + leading = (index == 0) + if ch == u' ': # space+ + spaces = True + else: # break+ + breaks = True + + # Series of whitespaces ended with a non-space. + elif spaces or breaks: + if leading: + if spaces and breaks: + mixed_breaks_spaces = True + elif spaces: + leading_spaces = True + elif breaks: + leading_breaks = True + else: + if mixed: + mixed_breaks_spaces = True + elif spaces and breaks: + inline_breaks_spaces = True + elif spaces: + inline_spaces = True + elif breaks: + inline_breaks = True + spaces = breaks = mixed = leading = False + + # Series of whitespaces reach the end. + if (spaces or breaks) and (index == len(scalar)-1): + if spaces and breaks: + mixed_breaks_spaces = True + elif spaces: + trailing_spaces = True + if leading: + leading_spaces = True + elif breaks: + trailing_breaks = True + if leading: + leading_breaks = True + spaces = breaks = mixed = leading = False + + # Prepare for the next character. + index += 1 + preceeded_by_space = (ch in u'\0 \t\r\n\x85\u2028\u2029') + followed_by_space = (index+1 >= len(scalar) or + scalar[index+1] in u'\0 \t\r\n\x85\u2028\u2029') + + # Let's decide what styles are allowed. + allow_flow_plain = True + allow_block_plain = True + allow_single_quoted = True + allow_double_quoted = True + allow_block = True + + # Leading and trailing whitespace are bad for plain scalars. We also + # do not want to mess with leading whitespaces for block scalars. + if leading_spaces or leading_breaks or trailing_spaces: + allow_flow_plain = allow_block_plain = allow_block = False + + # Trailing breaks are fine for block scalars, but unacceptable for + # plain scalars. + if trailing_breaks: + allow_flow_plain = allow_block_plain = False + + # The combination of (space+ break+) is only acceptable for block + # scalars. + if inline_breaks_spaces: + allow_flow_plain = allow_block_plain = allow_single_quoted = False + + # Mixed spaces and breaks, as well as special character are only + # allowed for double quoted scalars. + if mixed_breaks_spaces or special_characters: + allow_flow_plain = allow_block_plain = \ + allow_single_quoted = allow_block = False + + # We don't emit multiline plain scalars. + if line_breaks: + allow_flow_plain = allow_block_plain = False + + # Flow indicators are forbidden for flow plain scalars. + if flow_indicators: + allow_flow_plain = False + + # Block indicators are forbidden for block plain scalars. + if block_indicators: + allow_block_plain = False + + return ScalarAnalysis(scalar=scalar, + empty=False, multiline=line_breaks, + allow_flow_plain=allow_flow_plain, + allow_block_plain=allow_block_plain, + allow_single_quoted=allow_single_quoted, + allow_double_quoted=allow_double_quoted, + allow_block=allow_block) + + # Writers. + + def flush_stream(self): + if hasattr(self.stream, 'flush'): + self.stream.flush() + + def write_stream_start(self): + # Write BOM if needed. + if self.encoding and self.encoding.startswith('utf-16'): + self.stream.write(u'\xFF\xFE'.encode(self.encoding)) + + def write_stream_end(self): + self.flush_stream() + + def write_indicator(self, indicator, need_whitespace, + whitespace=False, indention=False): + if self.whitespace or not need_whitespace: + data = indicator + else: + data = u' '+indicator + self.whitespace = whitespace + self.indention = self.indention and indention + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_indent(self): + indent = self.indent or 0 + if not self.indention or self.column > indent \ + or (self.column == indent and not self.whitespace): + self.write_line_break() + if self.column < indent: + self.whitespace = True + data = u' '*(indent-self.column) + self.column = indent + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_line_break(self, data=None): + if data is None: + data = self.best_line_break + self.whitespace = True + self.indention = True + self.line += 1 + self.column = 0 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_version_directive(self, version_text): + data = u'%%YAML %s' % version_text + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + def write_tag_directive(self, handle_text, prefix_text): + data = u'%%TAG %s %s' % (handle_text, prefix_text) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + # Scalar streams. + + def write_single_quoted(self, text, split=True): + self.write_indicator(u'\'', True) + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch is None or ch != u' ': + if start+1 == end and self.column > self.best_width and split \ + and start != 0 and end != len(text): + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch is None or ch not in u'\n\x85\u2028\u2029': + if text[start] == u'\n': + self.write_line_break() + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + start = end + else: + if ch is None or ch in u' \n\x85\u2028\u2029' or ch == u'\'': + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch == u'\'': + data = u'\'\'' + self.column += 2 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + 1 + if ch is not None: + spaces = (ch == u' ') + breaks = (ch in u'\n\x85\u2028\u2029') + end += 1 + self.write_indicator(u'\'', False) + + ESCAPE_REPLACEMENTS = { + u'\0': u'0', + u'\x07': u'a', + u'\x08': u'b', + u'\x09': u't', + u'\x0A': u'n', + u'\x0B': u'v', + u'\x0C': u'f', + u'\x0D': u'r', + u'\x1B': u'e', + u'\"': u'\"', + u'\\': u'\\', + u'\x85': u'N', + u'\xA0': u'_', + u'\u2028': u'L', + u'\u2029': u'P', + } + + def write_double_quoted(self, text, split=True): + self.write_indicator(u'"', True) + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if ch is None or ch in u'"\\\x85\u2028\u2029\uFEFF' \ + or not (u'\x20' <= ch <= u'\x7E' + or (self.allow_unicode + and (u'\xA0' <= ch <= u'\uD7FF' + or u'\uE000' <= ch <= u'\uFFFD'))): + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + if ch in self.ESCAPE_REPLACEMENTS: + data = u'\\'+self.ESCAPE_REPLACEMENTS[ch] + elif ch <= u'\xFF': + data = u'\\x%02X' % ord(ch) + elif ch <= u'\uFFFF': + data = u'\\u%04X' % ord(ch) + else: + data = u'\\U%08X' % ord(ch) + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end+1 + if 0 < end < len(text)-1 and (ch == u' ' or start >= end) \ + and self.column+(end-start) > self.best_width and split: + data = text[start:end]+u'\\' + if start < end: + start = end + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_indent() + self.whitespace = False + self.indention = False + if text[start] == u' ': + data = u'\\' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + end += 1 + self.write_indicator(u'"', False) + + def determine_chomp(self, text): + tail = text[-2:] + while len(tail) < 2: + tail = u' '+tail + if tail[-1] in u'\n\x85\u2028\u2029': + if tail[-2] in u'\n\x85\u2028\u2029': + return u'+' + else: + return u'' + else: + return u'-' + + def write_folded(self, text): + chomp = self.determine_chomp(text) + self.write_indicator(u'>'+chomp, True) + self.write_indent() + leading_space = False + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in u'\n\x85\u2028\u2029': + if not leading_space and ch is not None and ch != u' ' \ + and text[start] == u'\n': + self.write_line_break() + leading_space = (ch == u' ') + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + elif spaces: + if ch != u' ': + if start+1 == end and self.column > self.best_width: + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + else: + if ch is None or ch in u' \n\x85\u2028\u2029': + data = text[start:end] + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in u'\n\x85\u2028\u2029') + spaces = (ch == u' ') + end += 1 + + def write_literal(self, text): + chomp = self.determine_chomp(text) + self.write_indicator(u'|'+chomp, True) + self.write_indent() + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in u'\n\x85\u2028\u2029': + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + else: + if ch is None or ch in u'\n\x85\u2028\u2029': + data = text[start:end] + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in u'\n\x85\u2028\u2029') + end += 1 + + def write_plain(self, text, split=True): + if not text: + return + if not self.whitespace: + data = u' ' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.writespace = False + self.indention = False + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch != u' ': + if start+1 == end and self.column > self.best_width and split: + self.write_indent() + self.writespace = False + self.indention = False + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch not in u'\n\x85\u2028\u2029': + if text[start] == u'\n': + self.write_line_break() + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + self.whitespace = False + self.indention = False + start = end + else: + if ch is None or ch in u' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + spaces = (ch == u' ') + breaks = (ch in u'\n\x85\u2028\u2029') + end += 1 + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/error.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/error.py new file mode 100644 index 00000000..8fa916b2 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/error.py @@ -0,0 +1,75 @@ + +__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError'] + +class Mark: + + def __init__(self, name, index, line, column, buffer, pointer): + self.name = name + self.index = index + self.line = line + self.column = column + self.buffer = buffer + self.pointer = pointer + + def get_snippet(self, indent=4, max_length=75): + if self.buffer is None: + return None + head = '' + start = self.pointer + while start > 0 and self.buffer[start-1] not in u'\0\r\n\x85\u2028\u2029': + start -= 1 + if self.pointer-start > max_length/2-1: + head = ' ... ' + start += 5 + break + tail = '' + end = self.pointer + while end < len(self.buffer) and self.buffer[end] not in u'\0\r\n\x85\u2028\u2029': + end += 1 + if end-self.pointer > max_length/2-1: + tail = ' ... ' + end -= 5 + break + snippet = self.buffer[start:end].encode('utf-8') + return ' '*indent + head + snippet + tail + '\n' \ + + ' '*(indent+self.pointer-start+len(head)) + '^' + + def __str__(self): + snippet = self.get_snippet() + where = " in \"%s\", line %d, column %d" \ + % (self.name, self.line+1, self.column+1) + if snippet is not None: + where += ":\n"+snippet + return where + +class YAMLError(Exception): + pass + +class MarkedYAMLError(YAMLError): + + def __init__(self, context=None, context_mark=None, + problem=None, problem_mark=None, note=None): + self.context = context + self.context_mark = context_mark + self.problem = problem + self.problem_mark = problem_mark + self.note = note + + def __str__(self): + lines = [] + if self.context is not None: + lines.append(self.context) + if self.context_mark is not None \ + and (self.problem is None or self.problem_mark is None + or self.context_mark.name != self.problem_mark.name + or self.context_mark.line != self.problem_mark.line + or self.context_mark.column != self.problem_mark.column): + lines.append(str(self.context_mark)) + if self.problem is not None: + lines.append(self.problem) + if self.problem_mark is not None: + lines.append(str(self.problem_mark)) + if self.note is not None: + lines.append(self.note) + return '\n'.join(lines) + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/events.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/events.py new file mode 100644 index 00000000..3f244fa0 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/events.py @@ -0,0 +1,86 @@ + +# Abstract classes. + +class Event: + def __init__(self, start_mark=None, end_mark=None): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in ['anchor', 'tag', 'implicit', 'value'] + if hasattr(self, key)] + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +class NodeEvent(Event): + def __init__(self, anchor, start_mark=None, end_mark=None): + self.anchor = anchor + self.start_mark = start_mark + self.end_mark = end_mark + +class CollectionStartEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None, + flow_style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class CollectionEndEvent(Event): + pass + +# Implementations. + +class StreamStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndEvent(Event): + pass + +class DocumentStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None, version=None, tags=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + self.version = version + self.tags = tags + +class DocumentEndEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + +class AliasEvent(NodeEvent): + pass + +class ScalarEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, value, + start_mark=None, end_mark=None, style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class SequenceStartEvent(CollectionStartEvent): + pass + +class SequenceEndEvent(CollectionEndEvent): + pass + +class MappingStartEvent(CollectionStartEvent): + pass + +class MappingEndEvent(CollectionEndEvent): + pass + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/loader.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/loader.py new file mode 100644 index 00000000..293ff467 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/loader.py @@ -0,0 +1,40 @@ + +__all__ = ['BaseLoader', 'SafeLoader', 'Loader'] + +from reader import * +from scanner import * +from parser import * +from composer import * +from constructor import * +from resolver import * + +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/nodes.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/nodes.py new file mode 100644 index 00000000..cb8c1cba --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/nodes.py @@ -0,0 +1,49 @@ + +class Node: + def __init__(self, tag, value, start_mark, end_mark): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + value = self.value + #if isinstance(value, list): + # if len(value) == 0: + # value = '<empty>' + # elif len(value) == 1: + # value = '<1 item>' + # else: + # value = '<%d items>' % len(value) + #else: + # if len(value) > 75: + # value = repr(value[:70]+u' ... ') + # else: + # value = repr(value) + value = repr(value) + return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value) + +class ScalarNode(Node): + id = 'scalar' + def __init__(self, tag, value, + start_mark=None, end_mark=None, style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class CollectionNode(Node): + def __init__(self, tag, value, + start_mark=None, end_mark=None, flow_style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class SequenceNode(CollectionNode): + id = 'sequence' + +class MappingNode(CollectionNode): + id = 'mapping' + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/parser.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/parser.py new file mode 100644 index 00000000..2aec0fe3 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/parser.py @@ -0,0 +1,484 @@ + +# YAML can be parsed by an LL(1) parser! +# +# We use the following production rules: +# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END? +# implicit_document ::= block_node DOCUMENT-END? +# block_node ::= ALIAS | properties? block_content +# flow_node ::= ALIAS | properties? flow_content +# properties ::= TAG ANCHOR? | ANCHOR TAG? +# block_content ::= block_collection | flow_collection | SCALAR +# flow_content ::= flow_collection | SCALAR +# block_collection ::= block_sequence | block_mapping +# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +# block_mapping ::= BLOCK-MAPPING_START ((KEY block_node_or_indentless_sequence?)? (VALUE block_node_or_indentless_sequence?)?)* BLOCK-END +# block_node_or_indentless_sequence ::= ALIAS | properties? (block_content | indentless_block_sequence) +# indentless_block_sequence ::= (BLOCK-ENTRY block_node?)+ +# flow_collection ::= flow_sequence | flow_mapping +# flow_sequence ::= FLOW-SEQUENCE-START (flow_sequence_entry FLOW-ENTRY)* flow_sequence_entry? FLOW-SEQUENCE-END +# flow_mapping ::= FLOW-MAPPING-START (flow_mapping_entry FLOW-ENTRY)* flow_mapping_entry? FLOW-MAPPING-END +# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + +# TODO: support for BOM within a stream. +# stream ::= (BOM? implicit_document)? (BOM? explicit_document)* STREAM-END + +# FIRST sets: +# stream: { STREAM-START } +# explicit_document: { DIRECTIVE DOCUMENT-START } +# implicit_document: FIRST(block_node) +# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_sequence: { BLOCK-SEQUENCE-START } +# block_mapping: { BLOCK-MAPPING-START } +# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY } +# indentless_sequence: { ENTRY } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_sequence: { FLOW-SEQUENCE-START } +# flow_mapping: { FLOW-MAPPING-START } +# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } +# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } + +__all__ = ['Parser', 'ParserError'] + +from error import MarkedYAMLError +from tokens import * +from events import * +from scanner import * + +class ParserError(MarkedYAMLError): + pass + +class Parser: + # Since writing a recursive-descendant parser is a straightforward task, we + # do not give many comments here. + # Note that we use Python generators. If you rewrite the parser in another + # language, you may replace all 'yield'-s with event handler calls. + + DEFAULT_TAGS = { + u'!': u'!', + u'!!': u'tag:yaml.org,2002:', + } + + def __init__(self): + self.current_event = None + self.yaml_version = None + self.tag_handles = {} + self.event_generator = self.parse_stream() + + def check_event(self, *choices): + # Check the type of the next event. + if self.current_event is None: + try: + self.current_event = self.event_generator.next() + except StopIteration: + pass + if self.current_event is not None: + if not choices: + return True + for choice in choices: + if isinstance(self.current_event, choice): + return True + return False + + def peek_event(self): + # Get the next event. + if self.current_event is None: + try: + self.current_event = self.event_generator.next() + except StopIteration: + pass + return self.current_event + + def get_event(self): + # Get the next event. + if self.current_event is None: + try: + self.current_event = self.event_generator.next() + except StopIteration: + pass + value = self.current_event + self.current_event = None + return value + + def __iter__(self): + # Iterator protocol. + return self.event_generator + + def parse_stream(self): + # STREAM-START implicit_document? explicit_document* STREAM-END + + # Parse start of stream. + token = self.get_token() + yield StreamStartEvent(token.start_mark, token.end_mark, + encoding=token.encoding) + + # Parse implicit document. + if not self.check_token(DirectiveToken, DocumentStartToken, + StreamEndToken): + self.tag_handles = self.DEFAULT_TAGS + token = self.peek_token() + start_mark = end_mark = token.start_mark + yield DocumentStartEvent(start_mark, end_mark, + explicit=False) + for event in self.parse_block_node(): + yield event + token = self.peek_token() + start_mark = end_mark = token.start_mark + explicit = False + while self.check_token(DocumentEndToken): + token = self.get_token() + end_mark = token.end_mark + explicit = True + yield DocumentEndEvent(start_mark, end_mark, + explicit=explicit) + + # Parse explicit documents. + while not self.check_token(StreamEndToken): + token = self.peek_token() + start_mark = token.start_mark + version, tags = self.process_directives() + if not self.check_token(DocumentStartToken): + raise ParserError(None, None, + "expected '<document start>', but found %r" + % self.peek_token().id, + self.peek_token().start_mark) + token = self.get_token() + end_mark = token.end_mark + yield DocumentStartEvent(start_mark, end_mark, + explicit=True, version=version, tags=tags) + if self.check_token(DirectiveToken, + DocumentStartToken, DocumentEndToken, StreamEndToken): + yield self.process_empty_scalar(token.end_mark) + else: + for event in self.parse_block_node(): + yield event + token = self.peek_token() + start_mark = end_mark = token.start_mark + explicit = False + while self.check_token(DocumentEndToken): + token = self.get_token() + end_mark = token.end_mark + explicit=True + yield DocumentEndEvent(start_mark, end_mark, + explicit=explicit) + + # Parse end of stream. + token = self.get_token() + yield StreamEndEvent(token.start_mark, token.end_mark) + + def process_directives(self): + # DIRECTIVE* + self.yaml_version = None + self.tag_handles = {} + while self.check_token(DirectiveToken): + token = self.get_token() + if token.name == u'YAML': + if self.yaml_version is not None: + raise ParserError(None, None, + "found duplicate YAML directive", token.start_mark) + major, minor = token.value + if major != 1: + raise ParserError(None, None, + "found incompatible YAML document (version 1.* is required)", + token.start_mark) + self.yaml_version = token.value + elif token.name == u'TAG': + handle, prefix = token.value + if handle in self.tag_handles: + raise ParserError(None, None, + "duplicate tag handle %r" % handle.encode('utf-8'), + token.start_mark) + self.tag_handles[handle] = prefix + if self.tag_handles: + value = self.yaml_version, self.tag_handles.copy() + else: + value = self.yaml_version, None + for key in self.DEFAULT_TAGS: + if key not in self.tag_handles: + self.tag_handles[key] = self.DEFAULT_TAGS[key] + return value + + def parse_block_node(self): + return self.parse_node(block=True) + + def parse_flow_node(self): + return self.parse_node() + + def parse_block_node_or_indentless_sequence(self): + return self.parse_node(block=True, indentless_sequence=True) + + def parse_node(self, block=False, indentless_sequence=False): + # block_node ::= ALIAS | properties? block_content + # flow_node ::= ALIAS | properties? flow_content + # properties ::= TAG ANCHOR? | ANCHOR TAG? + # block_content ::= block_collection | flow_collection | SCALAR + # flow_content ::= flow_collection | SCALAR + # block_collection ::= block_sequence | block_mapping + # block_node_or_indentless_sequence ::= ALIAS | properties? + # (block_content | indentless_block_sequence) + if self.check_token(AliasToken): + token = self.get_token() + yield AliasEvent(token.value, token.start_mark, token.end_mark) + else: + anchor = None + tag = None + start_mark = end_mark = tag_mark = None + if self.check_token(AnchorToken): + token = self.get_token() + start_mark = token.start_mark + end_mark = token.end_mark + anchor = token.value + if self.check_token(TagToken): + token = self.get_token() + tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + elif self.check_token(TagToken): + token = self.get_token() + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + if self.check_token(AnchorToken): + token = self.get_token() + end_mark = token.end_mark + anchor = token.value + if tag is not None and tag != u'!': + handle, suffix = tag + if handle is not None: + if handle not in self.tag_handles: + raise ParserError("while parsing a node", start_mark, + "found undefined tag handle %r" % handle.encode('utf-8'), + tag_mark) + tag = self.tag_handles[handle]+suffix + else: + tag = suffix + #if tag == u'!': + # raise ParserError("while parsing a node", start_mark, + # "found non-specific tag '!'", tag_mark, + # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.") + if start_mark is None: + start_mark = end_mark = self.peek_token().start_mark + event = None + collection_events = None + implicit = (tag is None or tag == u'!') + if indentless_sequence and self.check_token(BlockEntryToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark) + collection_events = self.parse_indentless_sequence() + else: + if self.check_token(ScalarToken): + token = self.get_token() + end_mark = token.end_mark + if (token.plain and tag is None) or tag == u'!': + implicit = (True, False) + elif tag is None: + implicit = (False, True) + else: + implicit = (False, False) + event = ScalarEvent(anchor, tag, implicit, token.value, + start_mark, end_mark, style=token.style) + elif self.check_token(FlowSequenceStartToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + collection_events = self.parse_flow_sequence() + elif self.check_token(FlowMappingStartToken): + end_mark = self.peek_token().end_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + collection_events = self.parse_flow_mapping() + elif block and self.check_token(BlockSequenceStartToken): + end_mark = self.peek_token().start_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + collection_events = self.parse_block_sequence() + elif block and self.check_token(BlockMappingStartToken): + end_mark = self.peek_token().start_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + collection_events = self.parse_block_mapping() + elif anchor is not None or tag is not None: + # Empty scalars are allowed even if a tag or an anchor is + # specified. + event = ScalarEvent(anchor, tag, (implicit, False), u'', + start_mark, end_mark) + else: + if block: + node = 'block' + else: + node = 'flow' + token = self.peek_token() + raise ParserError("while scanning a %s node" % node, start_mark, + "expected the node content, but found %r" % token.id, + token.start_mark) + yield event + if collection_events is not None: + for event in collection_events: + yield event + + def parse_block_sequence(self): + # BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END + token = self.get_token() + start_mark = token.start_mark + while self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, BlockEndToken): + for event in self.parse_block_node(): + yield event + else: + yield self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while scanning a block collection", start_mark, + "expected <block end>, but found %r" % token.id, token.start_mark) + token = self.get_token() + yield SequenceEndEvent(token.start_mark, token.end_mark) + + def parse_indentless_sequence(self): + # (BLOCK-ENTRY block_node?)+ + while self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, + KeyToken, ValueToken, BlockEndToken): + for event in self.parse_block_node(): + yield event + else: + yield self.process_empty_scalar(token.end_mark) + token = self.peek_token() + yield SequenceEndEvent(token.start_mark, token.start_mark) + + def parse_block_mapping(self): + # BLOCK-MAPPING_START + # ((KEY block_node_or_indentless_sequence?)? + # (VALUE block_node_or_indentless_sequence?)?)* + # BLOCK-END + token = self.get_token() + start_mark = token.start_mark + while self.check_token(KeyToken, ValueToken): + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + for event in self.parse_block_node_or_indentless_sequence(): + yield event + else: + yield self.process_empty_scalar(token.end_mark) + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + for event in self.parse_block_node_or_indentless_sequence(): + yield event + else: + yield self.process_empty_scalar(token.end_mark) + else: + token = self.peek_token() + yield self.process_empty_scalar(token.start_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while scanning a block mapping", start_mark, + "expected <block end>, but found %r" % token.id, token.start_mark) + token = self.get_token() + yield MappingEndEvent(token.start_mark, token.end_mark) + + def parse_flow_sequence(self): + # flow_sequence ::= FLOW-SEQUENCE-START + # (flow_sequence_entry FLOW-ENTRY)* + # flow_sequence_entry? + # FLOW-SEQUENCE-END + # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + # + # Note that while production rules for both flow_sequence_entry and + # flow_mapping_entry are equal, their interpretations are different. + # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?` + # generate an inline mapping (set syntax). + token = self.get_token() + start_mark = token.start_mark + while not self.check_token(FlowSequenceEndToken): + if self.check_token(KeyToken): + token = self.get_token() + yield MappingStartEvent(None, None, True, + token.start_mark, token.end_mark, + flow_style=True) + if not self.check_token(ValueToken, + FlowEntryToken, FlowSequenceEndToken): + for event in self.parse_flow_node(): + yield event + else: + yield self.process_empty_scalar(token.end_mark) + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowSequenceEndToken): + for event in self.parse_flow_node(): + yield event + else: + yield self.process_empty_scalar(token.end_mark) + else: + token = self.peek_token() + yield self.process_empty_scalar(token.start_mark) + token = self.peek_token() + yield MappingEndEvent(token.start_mark, token.start_mark) + else: + for event in self.parse_flow_node(): + yield event + if not self.check_token(FlowEntryToken, FlowSequenceEndToken): + token = self.peek_token() + raise ParserError("while scanning a flow sequence", start_mark, + "expected ',' or ']', but got %r" % token.id, token.start_mark) + if self.check_token(FlowEntryToken): + self.get_token() + token = self.get_token() + yield SequenceEndEvent(token.start_mark, token.end_mark) + + def parse_flow_mapping(self): + # flow_mapping ::= FLOW-MAPPING-START + # (flow_mapping_entry FLOW-ENTRY)* + # flow_mapping_entry? + # FLOW-MAPPING-END + # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + token = self.get_token() + start_mark = token.start_mark + while not self.check_token(FlowMappingEndToken): + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowMappingEndToken): + for event in self.parse_flow_node(): + yield event + else: + yield self.process_empty_scalar(token.end_mark) + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowMappingEndToken): + for event in self.parse_flow_node(): + yield event + else: + yield self.process_empty_scalar(token.end_mark) + else: + token = self.peek_token() + yield self.process_empty_scalar(token.start_mark) + else: + for event in self.parse_flow_node(): + yield event + yield self.process_empty_scalar(self.peek_token().start_mark) + if not self.check_token(FlowEntryToken, FlowMappingEndToken): + token = self.peek_token() + raise ParserError("while scanning a flow mapping", start_mark, + "expected ',' or '}', but got %r" % token.id, token.start_mark) + if self.check_token(FlowEntryToken): + self.get_token() + if not self.check_token(FlowMappingEndToken): + token = self.peek_token() + raise ParserError("while scanning a flow mapping", start_mark, + "expected '}', but found %r" % token.id, token.start_mark) + token = self.get_token() + yield MappingEndEvent(token.start_mark, token.end_mark) + + def process_empty_scalar(self, mark): + return ScalarEvent(None, None, (True, False), u'', mark, mark) + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/reader.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/reader.py new file mode 100644 index 00000000..beb76d0a --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/reader.py @@ -0,0 +1,222 @@ +# This module contains abstractions for the input stream. You don't have to +# looks further, there are no pretty code. +# +# We define two classes here. +# +# Mark(source, line, column) +# It's just a record and its only use is producing nice error messages. +# Parser does not use it for any other purposes. +# +# Reader(source, data) +# Reader determines the encoding of `data` and converts it to unicode. +# Reader provides the following methods and attributes: +# reader.peek(length=1) - return the next `length` characters +# reader.forward(length=1) - move the current position to `length` characters. +# reader.index - the number of the current character. +# reader.line, stream.column - the line and the column of the current character. + +__all__ = ['Reader', 'ReaderError'] + +from error import YAMLError, Mark + +import codecs, re + +# Unfortunately, codec functions in Python 2.3 does not support the `finish` +# arguments, so we have to write our own wrappers. + +try: + codecs.utf_8_decode('', 'strict', False) + from codecs import utf_8_decode, utf_16_le_decode, utf_16_be_decode + +except TypeError: + + def utf_16_le_decode(data, errors, finish=False): + if not finish and len(data) % 2 == 1: + data = data[:-1] + return codecs.utf_16_le_decode(data, errors) + + def utf_16_be_decode(data, errors, finish=False): + if not finish and len(data) % 2 == 1: + data = data[:-1] + return codecs.utf_16_be_decode(data, errors) + + def utf_8_decode(data, errors, finish=False): + if not finish: + # We are trying to remove a possible incomplete multibyte character + # from the suffix of the data. + # The first byte of a multi-byte sequence is in the range 0xc0 to 0xfd. + # All further bytes are in the range 0x80 to 0xbf. + # UTF-8 encoded UCS characters may be up to six bytes long. + count = 0 + while count < 5 and count < len(data) \ + and '\x80' <= data[-count-1] <= '\xBF': + count -= 1 + if count < 5 and count < len(data) \ + and '\xC0' <= data[-count-1] <= '\xFD': + data = data[:-count-1] + return codecs.utf_8_decode(data, errors) + +class ReaderError(YAMLError): + + def __init__(self, name, position, character, encoding, reason): + self.name = name + self.character = character + self.position = position + self.encoding = encoding + self.reason = reason + + def __str__(self): + if isinstance(self.character, str): + return "'%s' codec can't decode byte #x%02x: %s\n" \ + " in \"%s\", position %d" \ + % (self.encoding, ord(self.character), self.reason, + self.name, self.position) + else: + return "unacceptable character #x%04x: %s\n" \ + " in \"%s\", position %d" \ + % (ord(self.character), self.reason, + self.name, self.position) + +class Reader: + # Reader: + # - determines the data encoding and converts it to unicode, + # - checks if characters are in allowed range, + # - adds '\0' to the end. + + # Reader accepts + # - a `str` object, + # - a `unicode` object, + # - a file-like object with its `read` method returning `str`, + # - a file-like object with its `read` method returning `unicode`. + + # Yeah, it's ugly and slow. + + def __init__(self, stream): + self.name = None + self.stream = None + self.stream_pointer = 0 + self.eof = True + self.buffer = u'' + self.pointer = 0 + self.raw_buffer = None + self.raw_decode = None + self.encoding = None + self.index = 0 + self.line = 0 + self.column = 0 + if isinstance(stream, unicode): + self.name = "<unicode string>" + self.check_printable(stream) + self.buffer = stream+u'\0' + elif isinstance(stream, str): + self.name = "<string>" + self.raw_buffer = stream + self.determine_encoding() + else: + self.stream = stream + self.name = getattr(stream, 'name', "<file>") + self.eof = False + self.raw_buffer = '' + self.determine_encoding() + + def peek(self, index=0): + if self.pointer+index+1 >= len(self.buffer): + self.update(index+1) + return self.buffer[self.pointer+index] + + def prefix(self, length=1): + if self.pointer+length >= len(self.buffer): + self.update(length) + return self.buffer[self.pointer:self.pointer+length] + + def forward(self, length=1): + if self.pointer+length+1 >= len(self.buffer): + self.update(length+1) + for k in range(length): + ch = self.buffer[self.pointer] + self.pointer += 1 + self.index += 1 + if ch in u'\n\x85\u2028\u2029' \ + or (ch == u'\r' and self.buffer[self.pointer+1] != u'\n'): + self.line += 1 + self.column = 0 + elif ch != u'\uFEFF': + self.column += 1 + + def get_mark(self): + if self.stream is None: + return Mark(self.name, self.index, self.line, self.column, + self.buffer, self.pointer) + else: + return Mark(self.name, self.index, self.line, self.column, + None, None) + + def determine_encoding(self): + while not self.eof and len(self.raw_buffer) < 2: + self.update_raw() + if not isinstance(self.raw_buffer, unicode): + if self.raw_buffer.startswith(codecs.BOM_UTF16_LE): + self.raw_decode = utf_16_le_decode + self.encoding = 'utf-16-le' + elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE): + self.raw_decode = utf_16_be_decode + self.encoding = 'utf-16-be' + else: + self.raw_decode = utf_8_decode + self.encoding = 'utf-8' + self.update(1) + + NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]') + def check_printable(self, data): + match = self.NON_PRINTABLE.search(data) + if match: + character = match.group() + position = self.index+(len(self.buffer)-self.pointer)+match.start() + raise ReaderError(self.name, position, character, + 'unicode', "special characters are not allowed") + + def update(self, length): + if self.raw_buffer is None: + return + self.buffer = self.buffer[self.pointer:] + self.pointer = 0 + while len(self.buffer) < length: + if not self.eof: + self.update_raw() + if self.raw_decode is not None: + try: + data, converted = self.raw_decode(self.raw_buffer, + 'strict', self.eof) + except UnicodeDecodeError, exc: + character = exc.object[exc.start] + if self.stream is not None: + position = self.stream_pointer-len(self.raw_buffer)+exc.start + else: + position = exc.start + raise ReaderError(self.name, position, character, + exc.encoding, exc.reason) + else: + data = self.raw_buffer + converted = len(data) + self.check_printable(data) + self.buffer += data + self.raw_buffer = self.raw_buffer[converted:] + if self.eof: + self.buffer += u'\0' + self.raw_buffer = None + break + + def update_raw(self, size=1024): + data = self.stream.read(size) + if data: + self.raw_buffer += data + self.stream_pointer += len(data) + else: + self.eof = True + +#try: +# import psyco +# psyco.bind(Reader) +#except ImportError: +# pass + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/representer.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/representer.py new file mode 100644 index 00000000..cb37169d --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/representer.py @@ -0,0 +1,501 @@ + +__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer', + 'RepresenterError'] + +from error import * +from nodes import * + +try: + import datetime + datetime_available = True +except ImportError: + datetime_available = False + +try: + set +except NameError: + from sets import Set as set + +import sys, copy_reg + +class RepresenterError(YAMLError): + pass + +class BaseRepresenter: + + yaml_representers = {} + yaml_multi_representers = {} + + def __init__(self, default_style=None, default_flow_style=None): + self.default_style = default_style + self.default_flow_style = default_flow_style + self.represented_objects = {} + + def represent(self, data): + node = self.represent_data(data) + self.serialize(node) + self.represented_objects = {} + + class C: pass + c = C() + def f(): pass + classobj_type = type(C) + instance_type = type(c) + function_type = type(f) + builtin_function_type = type(abs) + module_type = type(sys) + del C, c, f + + def get_classobj_bases(self, cls): + bases = [cls] + for base in cls.__bases__: + bases.extend(self.get_classobj_bases(base)) + return bases + + def represent_data(self, data): + if self.ignore_aliases(data): + alias_key = None + else: + alias_key = id(data) + if alias_key is not None: + if alias_key in self.represented_objects: + node = self.represented_objects[alias_key] + if node is None: + raise RepresenterError("recursive objects are not allowed: %r" % data) + return node + self.represented_objects[alias_key] = None + data_types = type(data).__mro__ + if type(data) is self.instance_type: + data_types = self.get_classobj_bases(data.__class__)+list(data_types) + if data_types[0] in self.yaml_representers: + node = self.yaml_representers[data_types[0]](self, data) + else: + for data_type in data_types: + if data_type in self.yaml_multi_representers: + node = self.yaml_multi_representers[data_type](self, data) + break + else: + if None in self.yaml_multi_representers: + node = self.yaml_multi_representers[None](self, data) + elif None in self.yaml_representers: + node = self.yaml_representers[None](self, data) + else: + node = ScalarNode(None, unicode(data)) + if alias_key is not None: + self.represented_objects[alias_key] = node + return node + + def add_representer(cls, data_type, representer): + if not 'yaml_representers' in cls.__dict__: + cls.yaml_representers = cls.yaml_representers.copy() + cls.yaml_representers[data_type] = representer + add_representer = classmethod(add_representer) + + def add_multi_representer(cls, data_type, representer): + if not 'yaml_multi_representers' in cls.__dict__: + cls.yaml_multi_representers = cls.yaml_multi_representers.copy() + cls.yaml_multi_representers[data_type] = representer + add_multi_representer = classmethod(add_multi_representer) + + def represent_scalar(self, tag, value, style=None): + if style is None: + style = self.default_style + return ScalarNode(tag, value, style=style) + + def represent_sequence(self, tag, sequence, flow_style=None): + best_style = True + value = [] + for item in sequence: + node_item = self.represent_data(item) + if not (isinstance(node_item, ScalarNode) and not node_item.style): + best_style = False + value.append(self.represent_data(item)) + if flow_style is None: + flow_style = self.default_flow_style + if flow_style is None: + flow_style = best_style + return SequenceNode(tag, value, flow_style=flow_style) + + def represent_mapping(self, tag, mapping, flow_style=None): + best_style = True + if hasattr(mapping, 'keys'): + value = {} + for item_key in mapping.keys(): + item_value = mapping[item_key] + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, ScalarNode) and not node_key.style): + best_style = False + if not (isinstance(node_value, ScalarNode) and not node_value.style): + best_style = False + value[node_key] = node_value + else: + value = [] + for item_key, item_value in mapping: + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, ScalarNode) and not node_key.style): + best_style = False + if not (isinstance(node_value, ScalarNode) and not node_value.style): + best_style = False + value.append((node_key, node_value)) + if flow_style is None: + flow_style = self.default_flow_style + if flow_style is None: + flow_style = best_style + return MappingNode(tag, value, flow_style=flow_style) + + def ignore_aliases(self, data): + return False + +class SafeRepresenter(BaseRepresenter): + + def ignore_aliases(self, data): + if data in [None, ()]: + return True + if isinstance(data, (str, unicode, bool, int, float)): + return True + + def represent_none(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:null', + u'null') + + def represent_str(self, data): + tag = None + style = None + try: + data = unicode(data, 'ascii') + tag = u'tag:yaml.org,2002:str' + except UnicodeDecodeError: + try: + data = unicode(data, 'utf-8') + tag = u'tag:yaml.org,2002:str' + except UnicodeDecodeError: + data = data.encode('base64') + tag = u'tag:yaml.org,2002:binary' + style = '|' + return self.represent_scalar(tag, data, style=style) + + def represent_unicode(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:str', data) + + def represent_bool(self, data): + if data: + value = u'true' + else: + value = u'false' + return self.represent_scalar(u'tag:yaml.org,2002:bool', value) + + def represent_int(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data)) + + def represent_long(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data)) + + repr_pos_inf = repr(1e300000) + repr_neg_inf = repr(-1e300000) + repr_nan = repr(1e300000/1e300000) + + def represent_float(self, data): + repr_data = repr(data) + if repr_data == self.repr_pos_inf: + value = u'.inf' + elif repr_data == self.repr_neg_inf: + value = u'-.inf' + elif repr_data == self.repr_nan: + value = u'.nan' + else: + value = unicode(repr_data) + return self.represent_scalar(u'tag:yaml.org,2002:float', value) + + def represent_list(self, data): + pairs = (len(data) > 0 and isinstance(data, list)) + if pairs: + for item in data: + if not isinstance(item, tuple) or len(item) != 2: + pairs = False + break + if not pairs: + return self.represent_sequence(u'tag:yaml.org,2002:seq', data) + value = [] + for item_key, item_value in data: + value.append(self.represent_mapping(u'tag:yaml.org,2002:map', + [(item_key, item_value)])) + return SequenceNode(u'tag:yaml.org,2002:pairs', value) + + def represent_dict(self, data): + return self.represent_mapping(u'tag:yaml.org,2002:map', data) + + def represent_set(self, data): + value = {} + for key in data: + value[key] = None + return self.represent_mapping(u'tag:yaml.org,2002:set', value) + + def represent_date(self, data): + value = u'%04d-%02d-%02d' % (data.year, data.month, data.day) + return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value) + + def represent_datetime(self, data): + value = u'%04d-%02d-%02d %02d:%02d:%02d' \ + % (data.year, data.month, data.day, + data.hour, data.minute, data.second) + if data.microsecond: + value += u'.' + unicode(data.microsecond/1000000.0).split(u'.')[1] + if data.utcoffset(): + value += unicode(data.utcoffset()) + return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value) + + def represent_yaml_object(self, tag, data, cls, flow_style=None): + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__.copy() + if isinstance(state, dict): + state = state.items() + state.sort() + return self.represent_mapping(tag, state, flow_style=flow_style) + + def represent_undefined(self, data): + raise RepresenterError("cannot represent an object: %s" % data) + +SafeRepresenter.add_representer(type(None), + SafeRepresenter.represent_none) + +SafeRepresenter.add_representer(str, + SafeRepresenter.represent_str) + +SafeRepresenter.add_representer(unicode, + SafeRepresenter.represent_unicode) + +SafeRepresenter.add_representer(bool, + SafeRepresenter.represent_bool) + +SafeRepresenter.add_representer(int, + SafeRepresenter.represent_int) + +SafeRepresenter.add_representer(long, + SafeRepresenter.represent_long) + +SafeRepresenter.add_representer(float, + SafeRepresenter.represent_float) + +SafeRepresenter.add_representer(list, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(tuple, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(dict, + SafeRepresenter.represent_dict) + +SafeRepresenter.add_representer(set, + SafeRepresenter.represent_set) + +if datetime_available: + SafeRepresenter.add_representer(datetime.date, + SafeRepresenter.represent_date) + SafeRepresenter.add_representer(datetime.datetime, + SafeRepresenter.represent_datetime) + +SafeRepresenter.add_representer(None, + SafeRepresenter.represent_undefined) + +class Representer(SafeRepresenter): + + def represent_str(self, data): + tag = None + style = None + try: + data = unicode(data, 'ascii') + tag = u'tag:yaml.org,2002:str' + except UnicodeDecodeError: + try: + data = unicode(data, 'utf-8') + tag = u'tag:yaml.org,2002:python/str' + except UnicodeDecodeError: + data = data.encode('base64') + tag = u'tag:yaml.org,2002:binary' + style = '|' + return self.represent_scalar(tag, data, style=style) + + def represent_unicode(self, data): + tag = None + try: + data.encode('ascii') + tag = u'tag:yaml.org,2002:python/unicode' + except UnicodeEncodeError: + tag = u'tag:yaml.org,2002:str' + return self.represent_scalar(tag, data) + + def represent_long(self, data): + tag = u'tag:yaml.org,2002:int' + if int(data) is not data: + tag = u'tag:yaml.org,2002:python/long' + return self.represent_scalar(tag, unicode(data)) + + def represent_complex(self, data): + if data.imag == 0.0: + data = u'%r' % data.real + elif data.real == 0.0: + data = u'%rj' % data.imag + elif data.imag > 0: + data = u'%r+%rj' % (data.real, data.imag) + else: + data = u'%r%rj' % (data.real, data.imag) + return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data) + + def represent_tuple(self, data): + return self.represent_sequence(u'tag:yaml.org,2002:python/tuple', data) + + def represent_name(self, data): + name = u'%s.%s' % (data.__module__, data.__name__) + return self.represent_scalar(u'tag:yaml.org,2002:python/name:'+name, u'') + + def represent_module(self, data): + return self.represent_scalar( + u'tag:yaml.org,2002:python/module:'+data.__name__, u'') + + def represent_instance(self, data): + # For instances of classic classes, we use __getinitargs__ and + # __getstate__ to serialize the data. + + # If data.__getinitargs__ exists, the object must be reconstructed by + # calling cls(**args), where args is a tuple returned by + # __getinitargs__. Otherwise, the cls.__init__ method should never be + # called and the class instance is created by instantiating a trivial + # class and assigning to the instance's __class__ variable. + + # If data.__getstate__ exists, it returns the state of the object. + # Otherwise, the state of the object is data.__dict__. + + # We produce either a !!python/object or !!python/object/new node. + # If data.__getinitargs__ does not exist and state is a dictionary, we + # produce a !!python/object node . Otherwise we produce a + # !!python/object/new node. + + cls = data.__class__ + class_name = u'%s.%s' % (cls.__module__, cls.__name__) + args = None + state = None + if hasattr(data, '__getinitargs__'): + args = list(data.__getinitargs__()) + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__ + if args is None and isinstance(state, dict): + state = state.items() + state.sort() + return self.represent_mapping( + u'tag:yaml.org,2002:python/object:'+class_name, state) + if isinstance(state, dict) and not state: + return self.represent_sequence( + u'tag:yaml.org,2002:python/object/new:'+class_name, args) + value = {} + if args: + value['args'] = args + value['state'] = state + return self.represent_mapping( + u'tag:yaml.org,2002:python/object/new:'+class_name, value) + + def represent_object(self, data): + # We use __reduce__ API to save the data. data.__reduce__ returns + # a tuple of length 2-5: + # (function, args, state, listitems, dictitems) + + # For reconstructing, we calls function(*args), then set its state, + # listitems, and dictitems if they are not None. + + # A special case is when function.__name__ == '__newobj__'. In this + # case we create the object with args[0].__new__(*args). + + # Another special case is when __reduce__ returns a string - we don't + # support it. + + # We produce a !!python/object, !!python/object/new or + # !!python/object/apply node. + + cls = type(data) + if cls in copy_reg.dispatch_table: + reduce = copy_reg.dispatch_table[cls] + elif hasattr(data, '__reduce_ex__'): + reduce = data.__reduce_ex__(2) + elif hasattr(data, '__reduce__'): + reduce = data.__reduce__() + else: + raise RepresenterError("cannot represent object: %r" % data) + reduce = (list(reduce)+[None]*5)[:5] + function, args, state, listitems, dictitems = reduce + args = list(args) + if state is None: + state = {} + if listitems is not None: + listitems = list(listitems) + if dictitems is not None: + dictitems = dict(dictitems) + if function.__name__ == '__newobj__': + function = args[0] + args = args[1:] + tag = u'tag:yaml.org,2002:python/object/new:' + newobj = True + else: + tag = u'tag:yaml.org,2002:python/object/apply:' + newobj = False + function_name = u'%s.%s' % (function.__module__, function.__name__) + if not args and not listitems and not dictitems \ + and isinstance(state, dict) and newobj: + state = state.items() + state.sort() + return self.represent_mapping( + u'tag:yaml.org,2002:python/object:'+function_name, state) + if not listitems and not dictitems \ + and isinstance(state, dict) and not state: + return self.represent_sequence(tag+function_name, args) + value = {} + if args: + value['args'] = args + if state or not isinstance(state, dict): + value['state'] = state + if listitems: + value['listitems'] = listitems + if dictitems: + value['dictitems'] = dictitems + return self.represent_mapping(tag+function_name, value) + +Representer.add_representer(str, + Representer.represent_str) + +Representer.add_representer(unicode, + Representer.represent_unicode) + +Representer.add_representer(long, + Representer.represent_long) + +Representer.add_representer(complex, + Representer.represent_complex) + +Representer.add_representer(tuple, + Representer.represent_tuple) + +Representer.add_representer(type, + Representer.represent_name) + +Representer.add_representer(Representer.classobj_type, + Representer.represent_name) + +Representer.add_representer(Representer.function_type, + Representer.represent_name) + +Representer.add_representer(Representer.builtin_function_type, + Representer.represent_name) + +Representer.add_representer(Representer.module_type, + Representer.represent_module) + +Representer.add_multi_representer(Representer.instance_type, + Representer.represent_instance) + +Representer.add_multi_representer(object, + Representer.represent_object) + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/resolver.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/resolver.py new file mode 100644 index 00000000..7e580e98 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/resolver.py @@ -0,0 +1,205 @@ + +__all__ = ['BaseResolver', 'Resolver'] + +from error import * +from nodes import * + +import re + +class ResolverError(YAMLError): + pass + +class BaseResolver: + + DEFAULT_SCALAR_TAG = u'tag:yaml.org,2002:str' + DEFAULT_SEQUENCE_TAG = u'tag:yaml.org,2002:seq' + DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map' + + yaml_implicit_resolvers = {} + yaml_path_resolvers = {} + + def __init__(self): + self.resolver_exact_paths = [] + self.resolver_prefix_paths = [] + + def add_implicit_resolver(cls, tag, regexp, first): + if not 'yaml_implicit_resolvers' in cls.__dict__: + cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy() + if first is None: + first = [None] + for ch in first: + cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp)) + add_implicit_resolver = classmethod(add_implicit_resolver) + + def add_path_resolver(cls, tag, path, kind=None): + if not 'yaml_path_resolvers' in cls.__dict__: + cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy() + new_path = [] + for element in path: + if isinstance(element, (list, tuple)): + if len(element) == 2: + node_check, index_check = element + elif len(element) == 1: + node_check = element[0] + index_check = True + else: + raise ResolverError("Invalid path element: %s" % element) + else: + node_check = None + index_check = element + if node_check is str: + node_check = ScalarNode + elif node_check is list: + node_check = SequenceNode + elif node_check is map: + node_check = MappingNode + elif node_check not in [ScalarNode, SequenceNode, MappingNode] \ + and not isinstance(node_check, basestring) \ + and node_check is not None: + raise ResolverError("Invalid node checker: %s" % node_check) + if not isinstance(index_check, (basestring, int)) \ + and index_check is not None: + raise ResolverError("Invalid index checker: %s" % index_check) + new_path.append((node_check, index_check)) + if kind is str: + kind = ScalarNode + elif kind is list: + kind = SequenceNode + elif kind is map: + kind = MappingNode + elif kind not in [ScalarNode, SequenceNode, MappingNode] \ + and kind is not None: + raise ResolverError("Invalid node kind: %s" % kind) + cls.yaml_path_resolvers[tuple(new_path), kind] = tag + add_path_resolver = classmethod(add_path_resolver) + + def descend_resolver(self, current_node, current_index): + exact_paths = {} + prefix_paths = [] + if current_node: + depth = len(self.resolver_prefix_paths) + for path, kind in self.resolver_prefix_paths[-1]: + if self.check_resolver_prefix(depth, path, kind, + current_node, current_index): + if len(path) > depth: + prefix_paths.append((path, kind)) + else: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + for path, kind in self.yaml_path_resolvers: + if not path: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + prefix_paths.append((path, kind)) + self.resolver_exact_paths.append(exact_paths) + self.resolver_prefix_paths.append(prefix_paths) + + def ascend_resolver(self): + self.resolver_exact_paths.pop() + self.resolver_prefix_paths.pop() + + def check_resolver_prefix(self, depth, path, kind, + current_node, current_index): + node_check, index_check = path[depth-1] + if isinstance(node_check, basestring): + if current_node.tag != node_check: + return + elif node_check is not None: + if not isinstance(current_node, node_check): + return + if index_check is True and current_index is not None: + return + if index_check in [False, None] and current_index is None: + return + if isinstance(index_check, basestring): + if not (isinstance(current_index, ScalarNode) + and index_check == current_index.value): + return + elif isinstance(index_check, int): + if index_check != current_index: + return + return True + + def resolve(self, kind, value, implicit): + if kind is ScalarNode and implicit[0]: + if value == u'': + resolvers = self.yaml_implicit_resolvers.get(u'', []) + else: + resolvers = self.yaml_implicit_resolvers.get(value[0], []) + resolvers += self.yaml_implicit_resolvers.get(None, []) + for tag, regexp in resolvers: + if regexp.match(value): + return tag + implicit = implicit[1] + exact_paths = self.resolver_exact_paths[-1] + if kind in exact_paths: + return exact_paths[kind] + if None in exact_paths: + return exact_paths[None] + if kind is ScalarNode: + return self.DEFAULT_SCALAR_TAG + elif kind is SequenceNode: + return self.DEFAULT_SEQUENCE_TAG + elif kind is MappingNode: + return self.DEFAULT_MAPPING_TAG + +class Resolver(BaseResolver): + pass + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:bool', + re.compile(ur'''^(?:yes|Yes|YES|n|N|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF)$''', re.X), + list(u'yYnNtTfFoO')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:float', + re.compile(ur'''^(?:[-+]?(?:[0-9][0-9_]*)?\.[0-9_]*(?:[eE][-+][0-9]+)? + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list(u'-+0123456789.')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:int', + re.compile(ur'''^(?:[-+]?0b[0-1_]+ + |[-+]?0[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+ + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), + list(u'-+0123456789')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:merge', + re.compile(ur'^(?:<<)$'), + ['<']) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:null', + re.compile(ur'''^(?: ~ + |null|Null|NULL + | )$''', re.X), + [u'~', u'n', u'N', u'']) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:timestamp', + re.compile(ur'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? + (?:[Tt]|[ \t]+)[0-9][0-9]? + :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? + (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), + list(u'0123456789')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:value', + re.compile(ur'^(?:=)$'), + ['=']) + +# The following resolver is only for documentation purposes. It cannot work +# because plain scalars cannot start with '!', '&', or '*'. +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:yaml', + re.compile(ur'^(?:!|&|\*)$'), + list(u'!&*')) + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/scanner.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/scanner.py new file mode 100644 index 00000000..cf2478f9 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/scanner.py @@ -0,0 +1,1458 @@ + +# Scanner produces tokens of the following types: +# STREAM-START +# STREAM-END +# DIRECTIVE(name, value) +# DOCUMENT-START +# DOCUMENT-END +# BLOCK-SEQUENCE-START +# BLOCK-MAPPING-START +# BLOCK-END +# FLOW-SEQUENCE-START +# FLOW-MAPPING-START +# FLOW-SEQUENCE-END +# FLOW-MAPPING-END +# BLOCK-ENTRY +# FLOW-ENTRY +# KEY +# VALUE +# ALIAS(value) +# ANCHOR(value) +# TAG(value) +# SCALAR(value, plain) +# +# Read comments in the Scanner code for more details. +# + +__all__ = ['Scanner', 'ScannerError'] + +from error import MarkedYAMLError +from tokens import * + +class ScannerError(MarkedYAMLError): + pass + +class SimpleKey: + # See below simple keys treatment. + + def __init__(self, token_number, required, index, line, column, mark): + self.token_number = token_number + self.required = required + self.index = index + self.line = line + self.column = column + self.mark = mark + +class Scanner: + + def __init__(self): + """Initialize the scanner.""" + # It is assumed that Scanner and Reader will have a common descendant. + # Reader do the dirty work of checking for BOM and converting the + # input data to Unicode. It also adds NUL to the end. + # + # Reader supports the following methods + # self.peek(i=0) # peek the next i-th character + # self.prefix(l=1) # peek the next l characters + # self.forward(l=1) # read the next l characters and move the pointer. + + # Had we reached the end of the stream? + self.done = False + + # The number of unclosed '{' and '['. `flow_level == 0` means block + # context. + self.flow_level = 0 + + # List of processed tokens that are not yet emitted. + self.tokens = [] + + # Add the STREAM-START token. + self.fetch_stream_start() + + # Number of tokens that were emitted through the `get_token` method. + self.tokens_taken = 0 + + # The current indentation level. + self.indent = -1 + + # Past indentation levels. + self.indents = [] + + # Variables related to simple keys treatment. + + # A simple key is a key that is not denoted by the '?' indicator. + # Example of simple keys: + # --- + # block simple key: value + # ? not a simple key: + # : { flow simple key: value } + # We emit the KEY token before all keys, so when we find a potential + # simple key, we try to locate the corresponding ':' indicator. + # Simple keys should be limited to a single line and 1024 characters. + + # Can a simple key start at the current position? A simple key may + # start: + # - at the beginning of the line, not counting indentation spaces + # (in block context), + # - after '{', '[', ',' (in the flow context), + # - after '?', ':', '-' (in the block context). + # In the block context, this flag also signifies if a block collection + # may start at the current position. + self.allow_simple_key = True + + # Keep track of possible simple keys. This is a dictionary. The key + # is `flow_level`; there can be no more that one possible simple key + # for each level. The value is a SimpleKey record: + # (token_number, required, index, line, column, mark) + # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow), + # '[', or '{' tokens. + self.possible_simple_keys = {} + + # Public methods. + + def check_token(self, *choices): + # Check if the next token is one of the given types. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + if not choices: + return True + for choice in choices: + if isinstance(self.tokens[0], choice): + return True + return False + + def peek_token(self): + # Return the next token, but do not delete if from the queue. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + return self.tokens[0] + + def get_token(self): + # Return the next token. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + self.tokens_taken += 1 + return self.tokens.pop(0) + + def __iter__(self): + # Iterator protocol. + while self.need_more_tokens(): + self.fetch_more_tokens() + while self.tokens: + self.tokens_taken += 1 + yield self.tokens.pop(0) + while self.need_more_tokens(): + self.fetch_more_tokens() + + # Private methods. + + def need_more_tokens(self): + if self.done: + return False + if not self.tokens: + return True + # The current token may be a potential simple key, so we + # need to look further. + self.stale_possible_simple_keys() + if self.next_possible_simple_key() == self.tokens_taken: + return True + + def fetch_more_tokens(self): + + # Eat whitespaces and comments until we reach the next token. + self.scan_to_next_token() + + # Remove obsolete possible simple keys. + self.stale_possible_simple_keys() + + # Compare the current indentation and column. It may add some tokens + # and decrease the current indentation level. + self.unwind_indent(self.column) + + # Peek the next character. + ch = self.peek() + + # Is it the end of stream? + if ch == u'\0': + return self.fetch_stream_end() + + # Is it a directive? + if ch == u'%' and self.check_directive(): + return self.fetch_directive() + + # Is it the document start? + if ch == u'-' and self.check_document_start(): + return self.fetch_document_start() + + # Is it the document end? + if ch == u'.' and self.check_document_end(): + return self.fetch_document_end() + + # TODO: support for BOM within a stream. + #if ch == u'\uFEFF': + # return self.fetch_bom() <-- issue BOMToken + + # Note: the order of the following checks is NOT significant. + + # Is it the flow sequence start indicator? + if ch == u'[': + return self.fetch_flow_sequence_start() + + # Is it the flow mapping start indicator? + if ch == u'{': + return self.fetch_flow_mapping_start() + + # Is it the flow sequence end indicator? + if ch == u']': + return self.fetch_flow_sequence_end() + + # Is it the flow mapping end indicator? + if ch == u'}': + return self.fetch_flow_mapping_end() + + # Is it the flow entry indicator? + if ch in u',': + return self.fetch_flow_entry() + + # Is it the block entry indicator? + if ch in u'-' and self.check_block_entry(): + return self.fetch_block_entry() + + # Is it the key indicator? + if ch == u'?' and self.check_key(): + return self.fetch_key() + + # Is it the value indicator? + if ch == u':' and self.check_value(): + return self.fetch_value() + + # Is it an alias? + if ch == u'*': + return self.fetch_alias() + + # Is it an anchor? + if ch == u'&': + return self.fetch_anchor() + + # Is it a tag? + if ch == u'!': + return self.fetch_tag() + + # Is it a literal scalar? + if ch == u'|' and not self.flow_level: + return self.fetch_literal() + + # Is it a folded scalar? + if ch == u'>' and not self.flow_level: + return self.fetch_folded() + + # Is it a single quoted scalar? + if ch == u'\'': + return self.fetch_single() + + # Is it a double quoted scalar? + if ch == u'\"': + return self.fetch_double() + + # It must be a plain scalar then. + if self.check_plain(): + return self.fetch_plain() + + # No? It's an error. Let's produce a nice error message. + raise ScannerError("while scanning for the next token", None, + "found character %r that cannot start any token" + % ch.encode('utf-8'), self.get_mark()) + + # Simple keys treatment. + + def next_possible_simple_key(self): + # Return the number of the nearest possible simple key. Actually we + # don't need to loop through the whole dictionary. We may replace it + # with the following code: + # if not self.possible_simple_keys: + # return None + # return self.possible_simple_keys[ + # min(self.possible_simple_keys.keys())].token_number + min_token_number = None + for level in self.possible_simple_keys: + key = self.possible_simple_keys[level] + if min_token_number is None or key.token_number < min_token_number: + min_token_number = key.token_number + return min_token_number + + def stale_possible_simple_keys(self): + # Remove entries that are no longer possible simple keys. According to + # the YAML specification, simple keys + # - should be limited to a single line, + # - should be no longer than 1024 characters. + # Disabling this procedure will allow simple keys of any length and + # height (may cause problems if indentation is broken though). + for level in self.possible_simple_keys.keys(): + key = self.possible_simple_keys[level] + if key.line != self.line \ + or self.index-key.index > 1024: + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not found expected ':'", self.get_mark()) + del self.possible_simple_keys[level] + + def save_possible_simple_key(self): + # The next token may start a simple key. We check if it's possible + # and save its position. This function is called for + # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'. + + # Check if a simple key is required at the current position. + required = not self.flow_level and self.indent == self.column + + # A simple key is required only if it is the first token in the current + # line. Therefore it is always allowed. + assert self.allow_simple_key or not required + + # The next token might be a simple key. Let's save it's number and + # position. + if self.allow_simple_key: + self.remove_possible_simple_key() + token_number = self.tokens_taken+len(self.tokens) + key = SimpleKey(token_number, required, + self.index, self.line, self.column, self.get_mark()) + self.possible_simple_keys[self.flow_level] = key + + def remove_possible_simple_key(self): + # Remove the saved possible key position at the current flow level. + if self.flow_level in self.possible_simple_keys: + key = self.possible_simple_keys[self.flow_level] + + # I don't think it's possible, but I could be wrong. + assert not key.required + #if key.required: + # raise ScannerError("while scanning a simple key", key.mark, + # "could not found expected ':'", self.get_mark()) + + # Indentation functions. + + def unwind_indent(self, column): + + ## In flow context, tokens should respect indentation. + ## Actually the condition should be `self.indent >= column` according to + ## the spec. But this condition will prohibit intuitively correct + ## constructions such as + ## key : { + ## } + #if self.flow_level and self.indent > column: + # raise ScannerError(None, None, + # "invalid intendation or unclosed '[' or '{'", + # self.get_mark()) + + # In the flow context, indentation is ignored. We make the scanner less + # restrictive then specification requires. + if self.flow_level: + return + + # In block context, we may need to issue the BLOCK-END tokens. + while self.indent > column: + mark = self.get_mark() + self.indent = self.indents.pop() + self.tokens.append(BlockEndToken(mark, mark)) + + def add_indent(self, column): + # Check if we need to increase indentation. + if self.indent < column: + self.indents.append(self.indent) + self.indent = column + return True + return False + + # Fetchers. + + def fetch_stream_start(self): + # We always add STREAM-START as the first token and STREAM-END as the + # last token. + + # Read the token. + mark = self.get_mark() + + # Add STREAM-START. + self.tokens.append(StreamStartToken(mark, mark, + encoding=self.encoding)) + + + def fetch_stream_end(self): + + # Set the current intendation to -1. + self.unwind_indent(-1) + + # Reset everything (not really needed). + self.allow_simple_key = False + self.possible_simple_keys = {} + + # Read the token. + mark = self.get_mark() + + # Add STREAM-END. + self.tokens.append(StreamEndToken(mark, mark)) + + # The steam is finished. + self.done = True + + def fetch_directive(self): + + # Set the current intendation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Scan and add DIRECTIVE. + self.tokens.append(self.scan_directive()) + + def fetch_document_start(self): + self.fetch_document_indicator(DocumentStartToken) + + def fetch_document_end(self): + self.fetch_document_indicator(DocumentEndToken) + + def fetch_document_indicator(self, TokenClass): + + # Set the current intendation to -1. + self.unwind_indent(-1) + + # Reset simple keys. Note that there could not be a block collection + # after '---'. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Add DOCUMENT-START or DOCUMENT-END. + start_mark = self.get_mark() + self.forward(3) + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_start(self): + self.fetch_flow_collection_start(FlowSequenceStartToken) + + def fetch_flow_mapping_start(self): + self.fetch_flow_collection_start(FlowMappingStartToken) + + def fetch_flow_collection_start(self, TokenClass): + + # '[' and '{' may start a simple key. + self.save_possible_simple_key() + + # Increase the flow level. + self.flow_level += 1 + + # Simple keys are allowed after '[' and '{'. + self.allow_simple_key = True + + # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_end(self): + self.fetch_flow_collection_end(FlowSequenceEndToken) + + def fetch_flow_mapping_end(self): + self.fetch_flow_collection_end(FlowMappingEndToken) + + def fetch_flow_collection_end(self, TokenClass): + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Decrease the flow level. + self.flow_level -= 1 + + # No simple keys after ']' or '}'. + self.allow_simple_key = False + + # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_entry(self): + + # Simple keys are allowed after ','. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add FLOW-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(FlowEntryToken(start_mark, end_mark)) + + def fetch_block_entry(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a new entry? + if not self.allow_simple_key: + raise ScannerError(None, None, + "sequence entries are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-SEQUENCE-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockSequenceStartToken(mark, mark)) + + # It's an error for the block entry to occur in the flow context, + # but we let the parser detect this. + else: + pass + + # Simple keys are allowed after '-'. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add BLOCK-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(BlockEntryToken(start_mark, end_mark)) + + def fetch_key(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a key (not nessesary a simple)? + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping keys are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-MAPPING-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after '?' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add KEY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(KeyToken(start_mark, end_mark)) + + def fetch_value(self): + + # Do we determine a simple key? + if self.flow_level in self.possible_simple_keys: + + # Add KEY. + key = self.possible_simple_keys[self.flow_level] + del self.possible_simple_keys[self.flow_level] + self.tokens.insert(key.token_number-self.tokens_taken, + KeyToken(key.mark, key.mark)) + + # If this key starts a new block mapping, we need to add + # BLOCK-MAPPING-START. + if not self.flow_level: + if self.add_indent(key.column): + self.tokens.insert(key.token_number-self.tokens_taken, + BlockMappingStartToken(key.mark, key.mark)) + + # There cannot be two simple keys one after another. + self.allow_simple_key = False + + # It must be a part of a complex key. + else: + + # Block context needs additional checks. + # (Do we really need them? They will be catched by the parser + # anyway.) + if not self.flow_level: + + # We are allowed to start a complex value if and only if + # we can start a simple key. + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping values are not allowed here", + self.get_mark()) + + # Simple keys are allowed after ':' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add VALUE. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(ValueToken(start_mark, end_mark)) + + def fetch_alias(self): + + # ALIAS could be a simple key. + self.save_possible_simple_key() + + # No simple keys after ALIAS. + self.allow_simple_key = False + + # Scan and add ALIAS. + self.tokens.append(self.scan_anchor(AliasToken)) + + def fetch_anchor(self): + + # ANCHOR could start a simple key. + self.save_possible_simple_key() + + # No simple keys after ANCHOR. + self.allow_simple_key = False + + # Scan and add ANCHOR. + self.tokens.append(self.scan_anchor(AnchorToken)) + + def fetch_tag(self): + + # TAG could start a simple key. + self.save_possible_simple_key() + + # No simple keys after TAG. + self.allow_simple_key = False + + # Scan and add TAG. + self.tokens.append(self.scan_tag()) + + def fetch_literal(self): + self.fetch_block_scalar(style='|') + + def fetch_folded(self): + self.fetch_block_scalar(style='>') + + def fetch_block_scalar(self, style): + + # A simple key may follow a block scalar. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Scan and add SCALAR. + self.tokens.append(self.scan_block_scalar(style)) + + def fetch_single(self): + self.fetch_flow_scalar(style='\'') + + def fetch_double(self): + self.fetch_flow_scalar(style='"') + + def fetch_flow_scalar(self, style): + + # A flow scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after flow scalars. + self.allow_simple_key = False + + # Scan and add SCALAR. + self.tokens.append(self.scan_flow_scalar(style)) + + def fetch_plain(self): + + # A plain scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after plain scalars. But note that `scan_plain` will + # change this flag if the scan is finished at the beginning of the + # line. + self.allow_simple_key = False + + # Scan and add SCALAR. May change `allow_simple_key`. + self.tokens.append(self.scan_plain()) + + # Checkers. + + def check_directive(self): + + # DIRECTIVE: ^ '%' ... + # The '%' indicator is already checked. + if self.column == 0: + return True + + def check_document_start(self): + + # DOCUMENT-START: ^ '---' (' '|'\n') + if self.column == 0: + if self.prefix(3) == u'---' \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return True + + def check_document_end(self): + + # DOCUMENT-END: ^ '...' (' '|'\n') + if self.column == 0: + if self.prefix(3) == u'...' \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return True + + def check_block_entry(self): + + # BLOCK-ENTRY: '-' (' '|'\n') + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + + def check_key(self): + + # KEY(flow context): '?' + if self.flow_level: + return True + + # KEY(block context): '?' (' '|'\n') + else: + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + + def check_value(self): + + # VALUE(flow context): ':' + if self.flow_level: + return True + + # VALUE(block context): ':' (' '|'\n') + else: + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + + def check_plain(self): + + # A plain scalar may start with any non-space character except: + # '-', '?', ':', ',', '[', ']', '{', '}', + # '#', '&', '*', '!', '|', '>', '\'', '\"', + # '%', '@', '`'. + # + # It may also start with + # '-', '?', ':' + # if it is followed by a non-space character. + # + # Note that we limit the last rule to the block context (except the + # '-' character) because we want the flow context to be space + # independent. + ch = self.peek() + return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \ + or (self.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029' + and (ch == u'-' or (not self.flow_level and ch in u'?:'))) + + # Scanners. + + def scan_to_next_token(self): + # We ignore spaces, line breaks and comments. + # If we find a line break in the block context, we set the flag + # `allow_simple_key` on. + # The byte order mark is stripped if it's the first character in the + # stream. We do not yet support BOM inside the stream as the + # specification requires. Any such mark will be considered as a part + # of the document. + # + # TODO: We need to make tab handling rules more sane. A good rule is + # Tabs cannot precede tokens + # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END, + # KEY(block), VALUE(block), BLOCK-ENTRY + # So the checking code is + # if <TAB>: + # self.allow_simple_keys = False + # We also need to add the check for `allow_simple_keys == True` to + # `unwind_indent` before issuing BLOCK-END. + # Scanners for block, flow, and plain scalars need to be modified. + + if self.index == 0 and self.peek() == u'\uFEFF': + self.forward() + found = False + while not found: + while self.peek() == u' ': + self.forward() + if self.peek() == u'#': + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + if self.scan_line_break(): + if not self.flow_level: + self.allow_simple_key = True + else: + found = True + + def scan_directive(self): + # See the specification for details. + start_mark = self.get_mark() + self.forward() + name = self.scan_directive_name(start_mark) + value = None + if name == u'YAML': + value = self.scan_yaml_directive_value(start_mark) + end_mark = self.get_mark() + elif name == u'TAG': + value = self.scan_tag_directive_value(start_mark) + end_mark = self.get_mark() + else: + end_mark = self.get_mark() + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + self.scan_directive_ignored_line(start_mark) + return DirectiveToken(name, value, start_mark, end_mark) + + def scan_directive_name(self, start_mark): + # See the specification for details. + length = 0 + ch = self.peek(length) + while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \ + or ch in u'-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + return value + + def scan_yaml_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + major = self.scan_yaml_directive_number(start_mark) + if self.peek() != '.': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or '.', but found %r" + % self.peek().encode('utf-8'), + self.get_mark()) + self.forward() + minor = self.scan_yaml_directive_number(start_mark) + if self.peek() not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or ' ', but found %r" + % self.peek().encode('utf-8'), + self.get_mark()) + return (major, minor) + + def scan_yaml_directive_number(self, start_mark): + # See the specification for details. + ch = self.peek() + if not (u'0' <= ch <= '9'): + raise ScannerError("while scanning a directive", start_mark, + "expected a digit, but found %r" % ch.encode('utf-8'), + self.get_mark()) + length = 0 + while u'0' <= self.peek(length) <= u'9': + length += 1 + value = int(self.prefix(length)) + self.forward(length) + return value + + def scan_tag_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + handle = self.scan_tag_directive_handle(start_mark) + while self.peek() == u' ': + self.forward() + prefix = self.scan_tag_directive_prefix(start_mark) + return (handle, prefix) + + def scan_tag_directive_handle(self, start_mark): + # See the specification for details. + value = self.scan_tag_handle('directive', start_mark) + ch = self.peek() + if ch != u' ': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch.encode('utf-8'), + self.get_mark()) + return value + + def scan_tag_directive_prefix(self, start_mark): + # See the specification for details. + value = self.scan_tag_uri('directive', start_mark) + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch.encode('utf-8'), + self.get_mark()) + return value + + def scan_directive_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + if self.peek() == u'#': + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in u'\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a comment or a line break, but found %r" + % ch.encode('utf-8'), self.get_mark()) + self.scan_line_break() + + def scan_anchor(self, TokenClass): + # The specification does not restrict characters for anchors and + # aliases. This may lead to problems, for instance, the document: + # [ *alias, value ] + # can be interpteted in two ways, as + # [ "value" ] + # and + # [ *alias , "value" ] + # Therefore we restrict aliases to numbers and ASCII letters. + start_mark = self.get_mark() + indicator = self.peek() + if indicator == '*': + name = 'alias' + else: + name = 'anchor' + self.forward() + length = 0 + ch = self.peek(length) + while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \ + or ch in u'-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in u'\0 \t\r\n\x85\u2028\u2029?:,]}%@`': + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + end_mark = self.get_mark() + return TokenClass(value, start_mark, end_mark) + + def scan_tag(self): + # See the specification for details. + start_mark = self.get_mark() + ch = self.peek(1) + if ch == u'<': + handle = None + self.forward(2) + suffix = self.scan_tag_uri('tag', start_mark) + if self.peek() != u'>': + raise ScannerError("while parsing a tag", start_mark, + "expected '>', but found %r" % self.peek().encode('utf-8'), + self.get_mark()) + self.forward() + elif ch in u'\0 \t\r\n\x85\u2028\u2029': + handle = None + suffix = u'!' + self.forward() + else: + length = 1 + use_handle = False + while ch not in u'\0 \r\n\x85\u2028\u2029': + if ch == u'!': + use_handle = True + break + length += 1 + ch = self.peek(length) + handle = u'!' + if use_handle: + handle = self.scan_tag_handle('tag', start_mark) + else: + handle = u'!' + self.forward() + suffix = self.scan_tag_uri('tag', start_mark) + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a tag", start_mark, + "expected ' ', but found %r" % ch.encode('utf-8'), + self.get_mark()) + value = (handle, suffix) + end_mark = self.get_mark() + return TagToken(value, start_mark, end_mark) + + def scan_block_scalar(self, style): + # See the specification for details. + + if style == '>': + folded = True + else: + folded = False + + chunks = [] + start_mark = self.get_mark() + + # Scan the header. + self.forward() + chomping, increment = self.scan_block_scalar_indicators(start_mark) + self.scan_block_scalar_ignored_line(start_mark) + + # Determine the indentation level and go to the first non-empty line. + min_indent = self.indent+1 + if min_indent < 1: + min_indent = 1 + if increment is None: + breaks, max_indent, end_mark = self.scan_block_scalar_indentation() + indent = max(min_indent, max_indent) + else: + indent = min_indent+increment-1 + breaks, end_mark = self.scan_block_scalar_breaks(indent) + line_break = u'' + + # Scan the inner part of the block scalar. + while self.column == indent and self.peek() != u'\0': + chunks.extend(breaks) + leading_non_space = self.peek() not in u' \t' + length = 0 + while self.peek(length) not in u'\0\r\n\x85\u2028\u2029': + length += 1 + chunks.append(self.prefix(length)) + self.forward(length) + line_break = self.scan_line_break() + breaks, end_mark = self.scan_block_scalar_breaks(indent) + if self.column == indent and self.peek() != u'\0': + + # Unfortunately, folding rules are ambiguous. + # + # This is the folding according to the specification: + + if folded and line_break == u'\n' \ + and leading_non_space and self.peek() not in u' \t': + if not breaks: + chunks.append(u' ') + else: + chunks.append(line_break) + + # This is Clark Evans's interpretation (also in the spec + # examples): + # + #if folded and line_break == u'\n': + # if not breaks: + # if self.peek() not in ' \t': + # chunks.append(u' ') + # else: + # chunks.append(line_break) + #else: + # chunks.append(line_break) + else: + break + + # Chomp the tail. + if chomping is not False: + chunks.append(line_break) + if chomping is True: + chunks.extend(breaks) + + # We are done. + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) + + def scan_block_scalar_indicators(self, start_mark): + # See the specification for details. + chomping = None + increment = None + ch = self.peek() + if ch in u'+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch in u'0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + elif ch in u'0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + ch = self.peek() + if ch in u'+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected chomping or indentation indicators, but found %r" + % ch.encode('utf-8'), self.get_mark()) + return chomping, increment + + def scan_block_scalar_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + if self.peek() == u'#': + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in u'\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected a comment or a line break, but found %r" + % ch.encode('utf-8'), self.get_mark()) + self.scan_line_break() + + def scan_block_scalar_indentation(self): + # See the specification for details. + chunks = [] + max_indent = 0 + end_mark = self.get_mark() + while self.peek() in u' \r\n\x85\u2028\u2029': + if self.peek() != u' ': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + else: + self.forward() + if self.column > max_indent: + max_indent = self.column + return chunks, max_indent, end_mark + + def scan_block_scalar_breaks(self, indent): + # See the specification for details. + chunks = [] + end_mark = self.get_mark() + while self.column < indent and self.peek() == u' ': + self.forward() + while self.peek() in u'\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + while self.column < indent and self.peek() == u' ': + self.forward() + return chunks, end_mark + + def scan_flow_scalar(self, style): + # See the specification for details. + # Note that we loose indentation rules for quoted scalars. Quoted + # scalars don't need to adhere indentation because " and ' clearly + # mark the beginning and the end of them. Therefore we are less + # restrictive then the specification requires. We only need to check + # that document separators are not included in scalars. + if style == '"': + double = True + else: + double = False + chunks = [] + start_mark = self.get_mark() + quote = self.peek() + self.forward() + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + while self.peek() != quote: + chunks.extend(self.scan_flow_scalar_spaces(double, start_mark)) + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + self.forward() + end_mark = self.get_mark() + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) + + ESCAPE_REPLACEMENTS = { + u'0': u'\0', + u'a': u'\x07', + u'b': u'\x08', + u't': u'\x09', + u'\t': u'\x09', + u'n': u'\x0A', + u'v': u'\x0B', + u'f': u'\x0C', + u'r': u'\x0D', + u'e': u'\x1B', + u' ': u'\x20', + u'\"': u'\"', + u'\\': u'\\', + u'N': u'\x85', + u'_': u'\xA0', + u'L': u'\u2028', + u'P': u'\u2029', + } + + ESCAPE_CODES = { + u'x': 2, + u'u': 4, + u'U': 8, + } + + def scan_flow_scalar_non_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + length = 0 + while self.peek(length) not in u'\'\"\\\0 \t\r\n\x85\u2028\u2029': + length += 1 + if length: + chunks.append(self.prefix(length)) + self.forward(length) + ch = self.peek() + if not double and ch == u'\'' and self.peek(1) == u'\'': + chunks.append(u'\'') + self.forward(2) + elif (double and ch == u'\'') or (not double and ch in u'\"\\'): + chunks.append(ch) + self.forward() + elif double and ch == u'\\': + self.forward() + ch = self.peek() + if ch in self.ESCAPE_REPLACEMENTS: + chunks.append(self.ESCAPE_REPLACEMENTS[ch]) + self.forward() + elif ch in self.ESCAPE_CODES: + length = self.ESCAPE_CODES[ch] + self.forward() + for k in range(length): + if self.peek(k) not in u'0123456789ABCDEFabcdef': + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "expected escape sequence of %d hexdecimal numbers, but found %r" % + (length, self.peek(k).encode('utf-8')), self.get_mark()) + code = int(self.prefix(length), 16) + chunks.append(unichr(code)) + self.forward(length) + elif ch in u'\r\n\x85\u2028\u2029': + self.scan_line_break() + chunks.extend(self.scan_flow_scalar_breaks(double, start_mark)) + else: + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "found unknown escape character %r" % ch.encode('utf-8'), self.get_mark()) + else: + return chunks + + def scan_flow_scalar_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + length = 0 + while self.peek(length) in u' \t': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch == u'\0': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected end of stream", self.get_mark()) + elif ch in u'\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + breaks = self.scan_flow_scalar_breaks(double, start_mark) + if line_break != u'\n': + chunks.append(line_break) + elif not breaks: + chunks.append(u' ') + chunks.extend(breaks) + else: + chunks.append(whitespaces) + return chunks + + def scan_flow_scalar_breaks(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + # Instead of checking indentation, we check for document + # separators. + prefix = self.prefix(3) + if (prefix == u'---' or prefix == u'...') \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected document separator", self.get_mark()) + while self.peek() in u' \t': + self.forward() + if self.peek() in u'\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + else: + return chunks + + def scan_plain(self): + # See the specification for details. + # We add an additional restriction for the flow context: + # plain scalars in the flow context cannot contain ',', ':' and '?'. + # We also keep track of the `allow_simple_key` flag here. + # Indentation rules are loosed for the flow context. + chunks = [] + start_mark = self.get_mark() + end_mark = start_mark + indent = self.indent+1 + # We allow zero indentation for scalars, but then we need to check for + # document separators at the beginning of the line. + #if indent == 0: + # indent = 1 + spaces = [] + while True: + length = 0 + if self.peek() == u'#': + break + while True: + ch = self.peek(length) + if ch in u'\0 \t\r\n\x85\u2028\u2029' \ + or (not self.flow_level and ch == u':' and + self.peek(length+1) in u'\0 \t\r\n\x28\u2028\u2029') \ + or (self.flow_level and ch in u',:?[]{}'): + break + length += 1 + # It's not clear what we should do with ':' in the flow context. + if (self.flow_level and ch == u':' + and self.peek(length+1) not in u'\0 \t\r\n\x28\u2028\u2029,[]{}'): + self.forward(length) + raise ScannerError("while scanning a plain scalar", start_mark, + "found unexpected ':'", self.get_mark(), + "Please check http://pyyaml.org/wiki/YAMLColonInFlowContext for details.") + if length == 0: + break + self.allow_simple_key = False + chunks.extend(spaces) + chunks.append(self.prefix(length)) + self.forward(length) + end_mark = self.get_mark() + spaces = self.scan_plain_spaces(indent, start_mark) + if not spaces or self.peek() == u'#' \ + or (not self.flow_level and self.column < indent): + break + return ScalarToken(u''.join(chunks), True, start_mark, end_mark) + + def scan_plain_spaces(self, indent, start_mark): + # See the specification for details. + # The specification is really confusing about tabs in plain scalars. + # We just forbid them completely. Do not use tabs in YAML! + chunks = [] + length = 0 + while self.peek(length) in u' ': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch in u'\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + self.allow_simple_key = True + prefix = self.prefix(3) + if (prefix == u'---' or prefix == u'...') \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return + breaks = [] + while self.peek() in u' \r\n\x85\u2028\u2029': + if self.peek() == ' ': + self.forward() + else: + breaks.append(self.scan_line_break()) + prefix = self.prefix(3) + if (prefix == u'---' or prefix == u'...') \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return + if line_break != u'\n': + chunks.append(line_break) + elif not breaks: + chunks.append(u' ') + chunks.extend(breaks) + elif whitespaces: + chunks.append(whitespaces) + return chunks + + def scan_tag_handle(self, name, start_mark): + # See the specification for details. + # For some strange reasons, the specification does not allow '_' in + # tag handles. I have allowed it anyway. + ch = self.peek() + if ch != u'!': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch.encode('utf-8'), + self.get_mark()) + length = 1 + ch = self.peek(length) + if ch != u' ': + while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \ + or ch in u'-_': + length += 1 + ch = self.peek(length) + if ch != u'!': + self.forward(length) + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch.encode('utf-8'), + self.get_mark()) + length += 1 + value = self.prefix(length) + self.forward(length) + return value + + def scan_tag_uri(self, name, start_mark): + # See the specification for details. + # Note: we do not check if URI is well-formed. + chunks = [] + length = 0 + ch = self.peek(length) + while u'0' <= ch <= u'9' or u'A' <= ch <= 'Z' or u'a' <= ch <= 'z' \ + or ch in u'-;/?:@&=+$,_.!~*\'()[]%': + if ch == u'%': + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + chunks.append(self.scan_uri_escapes(name, start_mark)) + else: + length += 1 + ch = self.peek(length) + if length: + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + if not chunks: + raise ScannerError("while parsing a %s" % name, start_mark, + "expected URI, but found %r" % ch.encode('utf-8'), + self.get_mark()) + return u''.join(chunks) + + def scan_uri_escapes(self, name, start_mark): + # See the specification for details. + bytes = [] + mark = self.get_mark() + while self.peek() == u'%': + self.forward() + for k in range(2): + if self.peek(k) not in u'0123456789ABCDEFabcdef': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected URI escape sequence of 2 hexdecimal numbers, but found %r" % + (self.peek(k).encode('utf-8')), self.get_mark()) + bytes.append(chr(int(self.prefix(2), 16))) + self.forward(2) + try: + value = unicode(''.join(bytes), 'utf-8') + except UnicodeDecodeError, exc: + raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark) + return value + + def scan_line_break(self): + # Transforms: + # '\r\n' : '\n' + # '\r' : '\n' + # '\n' : '\n' + # '\x85' : '\n' + # '\u2028' : '\u2028' + # '\u2029 : '\u2029' + # default : '' + ch = self.peek() + if ch in u'\r\n\x85': + if self.prefix(2) == u'\r\n': + self.forward(2) + else: + self.forward() + return u'\n' + elif ch in u'\u2028\u2029': + self.forward() + return ch + return u'' + +#try: +# import psyco +# psyco.bind(Scanner) +#except ImportError: +# pass + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/serializer.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/serializer.py new file mode 100644 index 00000000..937be9a9 --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/serializer.py @@ -0,0 +1,121 @@ + +__all__ = ['Serializer', 'SerializerError'] + +from error import YAMLError +from events import * +from nodes import * + +class SerializerError(YAMLError): + pass + +class Serializer: + + ANCHOR_TEMPLATE = u'id%03d' + + def __init__(self, encoding=None, + explicit_start=None, explicit_end=None, version=None, tags=None): + self.use_encoding = encoding + self.use_explicit_start = explicit_start + self.use_explicit_end = explicit_end + self.use_version = version + self.use_tags = tags + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + self.closed = None + + def open(self): + if self.closed is None: + self.emit(StreamStartEvent(encoding=self.use_encoding)) + self.closed = False + elif self.closed: + raise SerializerError("serializer is closed") + else: + raise SerializerError("serializer is already opened") + + def close(self): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif not self.closed: + self.emit(StreamEndEvent()) + self.closed = True + + #def __del__(self): + # self.close() + + def serialize(self, node): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif self.closed: + raise SerializerError("serializer is closed") + self.emit(DocumentStartEvent(explicit=self.use_explicit_start, + version=self.use_version, tags=self.use_tags)) + self.anchor_node(node) + self.serialize_node(node, None, None) + self.emit(DocumentEndEvent(explicit=self.use_explicit_end)) + self.serialized_nodes = {} + self.anchors = {} + self.last_alias_id = 0 + + def anchor_node(self, node): + if node in self.anchors: + if self.anchors[node] is None: + self.anchors[node] = self.generate_anchor(node) + else: + self.anchors[node] = None + if isinstance(node, SequenceNode): + for item in node.value: + self.anchor_node(item) + elif isinstance(node, MappingNode): + if hasattr(node.value, 'keys'): + for key in node.value.keys(): + self.anchor_node(key) + self.anchor_node(node.value[key]) + else: + for key, value in node.value: + self.anchor_node(key) + self.anchor_node(value) + + def generate_anchor(self, node): + self.last_anchor_id += 1 + return self.ANCHOR_TEMPLATE % self.last_anchor_id + + def serialize_node(self, node, parent, index): + alias = self.anchors[node] + if node in self.serialized_nodes: + self.emit(AliasEvent(alias)) + else: + self.serialized_nodes[node] = True + self.descend_resolver(parent, index) + if isinstance(node, ScalarNode): + detected_tag = self.resolve(ScalarNode, node.value, (True, False)) + default_tag = self.resolve(ScalarNode, node.value, (False, True)) + implicit = (node.tag == detected_tag), (node.tag == default_tag) + self.emit(ScalarEvent(alias, node.tag, implicit, node.value, + style=node.style)) + elif isinstance(node, SequenceNode): + implicit = (node.tag + == self.resolve(SequenceNode, node.value, True)) + self.emit(SequenceStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + index = 0 + for item in node.value: + self.serialize_node(item, node, index) + index += 1 + self.emit(SequenceEndEvent()) + elif isinstance(node, MappingNode): + implicit = (node.tag + == self.resolve(MappingNode, node.value, True)) + self.emit(MappingStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + if hasattr(node.value, 'keys'): + for key in node.value.keys(): + self.serialize_node(key, node, None) + self.serialize_node(node.value[key], node, key) + else: + for key, value in node.value: + self.serialize_node(key, node, None) + self.serialize_node(value, node, key) + self.emit(MappingEndEvent()) + self.ascend_resolver() + diff --git a/scripts/external_libs/PyYAML-3.01/lib/yaml/tokens.py b/scripts/external_libs/PyYAML-3.01/lib/yaml/tokens.py new file mode 100644 index 00000000..4fe4522e --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/lib/yaml/tokens.py @@ -0,0 +1,104 @@ + +class Token: + def __init__(self, start_mark, end_mark): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in self.__dict__ + if not key.endswith('_mark')] + attributes.sort() + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +#class BOMToken(Token): +# id = '<byte order mark>' + +class DirectiveToken(Token): + id = '<directive>' + def __init__(self, name, value, start_mark, end_mark): + self.name = name + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class DocumentStartToken(Token): + id = '<document start>' + +class DocumentEndToken(Token): + id = '<document end>' + +class StreamStartToken(Token): + id = '<stream start>' + def __init__(self, start_mark=None, end_mark=None, + encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndToken(Token): + id = '<stream end>' + +class BlockSequenceStartToken(Token): + id = '<block sequence start>' + +class BlockMappingStartToken(Token): + id = '<block mapping start>' + +class BlockEndToken(Token): + id = '<block end>' + +class FlowSequenceStartToken(Token): + id = '[' + +class FlowMappingStartToken(Token): + id = '{' + +class FlowSequenceEndToken(Token): + id = ']' + +class FlowMappingEndToken(Token): + id = '}' + +class KeyToken(Token): + id = '?' + +class ValueToken(Token): + id = ':' + +class BlockEntryToken(Token): + id = '-' + +class FlowEntryToken(Token): + id = ',' + +class AliasToken(Token): + id = '<alias>' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class AnchorToken(Token): + id = '<anchor>' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class TagToken(Token): + id = '<tag>' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class ScalarToken(Token): + id = '<scalar>' + def __init__(self, value, plain, start_mark, end_mark, style=None): + self.value = value + self.plain = plain + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + diff --git a/scripts/external_libs/PyYAML-3.01/setup.py b/scripts/external_libs/PyYAML-3.01/setup.py new file mode 100644 index 00000000..23c1efac --- /dev/null +++ b/scripts/external_libs/PyYAML-3.01/setup.py @@ -0,0 +1,52 @@ + +NAME = 'PyYAML' +VERSION = '3.01' +DESCRIPTION = "YAML parser and emitter for Python" +LONG_DESCRIPTION = """\ +YAML is a data serialization format designed for human readability and +interaction with scripting languages. PyYAML is a YAML parser and +emitter for Python. + +PyYAML features a complete YAML 1.1 parser, Unicode support, pickle +support, capable extension API, and sensible error messages. PyYAML +supports standard YAML tags and provides Python-specific tags that allow +to represent an arbitrary Python object. + +PyYAML is applicable for a broad range of tasks from complex +configuration files to object serialization and persistance.""" +AUTHOR = "Kirill Simonov" +AUTHOR_EMAIL = 'xi@resolvent.net' +LICENSE = "MIT" +PLATFORMS = "Any" +URL = "http://pyyaml.org/wiki/PyYAML" +DOWNLOAD_URL = "http://pyyaml.org/download/pyyaml/%s-%s.tar.gz" % (NAME, VERSION) +CLASSIFIERS = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Text Processing :: Markup", +] + + +from distutils.core import setup + +setup( + name=NAME, + version=VERSION, + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + author=AUTHOR, + author_email=AUTHOR_EMAIL, + license=LICENSE, + platforms=PLATFORMS, + url=URL, + download_url=DOWNLOAD_URL, + classifiers=CLASSIFIERS, + + package_dir={'': 'lib'}, + packages=['yaml'], +) + diff --git a/scripts/external_libs/__init__.py b/scripts/external_libs/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/scripts/external_libs/__init__.py @@ -0,0 +1 @@ + diff --git a/scripts/external_libs/enum34-1.0.4/PKG-INFO b/scripts/external_libs/enum34-1.0.4/PKG-INFO new file mode 100644 index 00000000..428ce0e3 --- /dev/null +++ b/scripts/external_libs/enum34-1.0.4/PKG-INFO @@ -0,0 +1,746 @@ +Metadata-Version: 1.1 +Name: enum34 +Version: 1.0.4 +Summary: Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4 +Home-page: https://pypi.python.org/pypi/enum34 +Author: Ethan Furman +Author-email: ethan@stoneleaf.us +License: BSD License +Description: ``enum`` --- support for enumerations + ======================================== + + .. :synopsis: enumerations are sets of symbolic names bound to unique, constant + values. + .. :moduleauthor:: Ethan Furman <ethan@stoneleaf.us> + .. :sectionauthor:: Barry Warsaw <barry@python.org>, + .. :sectionauthor:: Eli Bendersky <eliben@gmail.com>, + .. :sectionauthor:: Ethan Furman <ethan@stoneleaf.us> + + ---------------- + + An enumeration is a set of symbolic names (members) bound to unique, constant + values. Within an enumeration, the members can be compared by identity, and + the enumeration itself can be iterated over. + + + Module Contents + --------------- + + This module defines two enumeration classes that can be used to define unique + sets of names and values: ``Enum`` and ``IntEnum``. It also defines + one decorator, ``unique``. + + ``Enum`` + + Base class for creating enumerated constants. See section `Functional API`_ + for an alternate construction syntax. + + ``IntEnum`` + + Base class for creating enumerated constants that are also subclasses of ``int``. + + ``unique`` + + Enum class decorator that ensures only one name is bound to any one value. + + + Creating an Enum + ---------------- + + Enumerations are created using the ``class`` syntax, which makes them + easy to read and write. An alternative creation method is described in + `Functional API`_. To define an enumeration, subclass ``Enum`` as + follows:: + + >>> from enum import Enum + >>> class Color(Enum): + ... red = 1 + ... green = 2 + ... blue = 3 + + Note: Nomenclature + + - The class ``Color`` is an *enumeration* (or *enum*) + - The attributes ``Color.red``, ``Color.green``, etc., are + *enumeration members* (or *enum members*). + - The enum members have *names* and *values* (the name of + ``Color.red`` is ``red``, the value of ``Color.blue`` is + ``3``, etc.) + + Note: + + Even though we use the ``class`` syntax to create Enums, Enums + are not normal Python classes. See `How are Enums different?`_ for + more details. + + Enumeration members have human readable string representations:: + + >>> print(Color.red) + Color.red + + ...while their ``repr`` has more information:: + + >>> print(repr(Color.red)) + <Color.red: 1> + + The *type* of an enumeration member is the enumeration it belongs to:: + + >>> type(Color.red) + <enum 'Color'> + >>> isinstance(Color.green, Color) + True + >>> + + Enum members also have a property that contains just their item name:: + + >>> print(Color.red.name) + red + + Enumerations support iteration. In Python 3.x definition order is used; in + Python 2.x the definition order is not available, but class attribute + ``__order__`` is supported; otherwise, value order is used:: + + >>> class Shake(Enum): + ... __order__ = 'vanilla chocolate cookies mint' # only needed in 2.x + ... vanilla = 7 + ... chocolate = 4 + ... cookies = 9 + ... mint = 3 + ... + >>> for shake in Shake: + ... print(shake) + ... + Shake.vanilla + Shake.chocolate + Shake.cookies + Shake.mint + + The ``__order__`` attribute is always removed, and in 3.x it is also ignored + (order is definition order); however, in the stdlib version it will be ignored + but not removed. + + Enumeration members are hashable, so they can be used in dictionaries and sets:: + + >>> apples = {} + >>> apples[Color.red] = 'red delicious' + >>> apples[Color.green] = 'granny smith' + >>> apples == {Color.red: 'red delicious', Color.green: 'granny smith'} + True + + + Programmatic access to enumeration members and their attributes + --------------------------------------------------------------- + + Sometimes it's useful to access members in enumerations programmatically (i.e. + situations where ``Color.red`` won't do because the exact color is not known + at program-writing time). ``Enum`` allows such access:: + + >>> Color(1) + <Color.red: 1> + >>> Color(3) + <Color.blue: 3> + + If you want to access enum members by *name*, use item access:: + + >>> Color['red'] + <Color.red: 1> + >>> Color['green'] + <Color.green: 2> + + If have an enum member and need its ``name`` or ``value``:: + + >>> member = Color.red + >>> member.name + 'red' + >>> member.value + 1 + + + Duplicating enum members and values + ----------------------------------- + + Having two enum members (or any other attribute) with the same name is invalid; + in Python 3.x this would raise an error, but in Python 2.x the second member + simply overwrites the first:: + + >>> # python 2.x + >>> class Shape(Enum): + ... square = 2 + ... square = 3 + ... + >>> Shape.square + <Shape.square: 3> + + >>> # python 3.x + >>> class Shape(Enum): + ... square = 2 + ... square = 3 + Traceback (most recent call last): + ... + TypeError: Attempted to reuse key: 'square' + + However, two enum members are allowed to have the same value. Given two members + A and B with the same value (and A defined first), B is an alias to A. By-value + lookup of the value of A and B will return A. By-name lookup of B will also + return A:: + + >>> class Shape(Enum): + ... __order__ = 'square diamond circle alias_for_square' # only needed in 2.x + ... square = 2 + ... diamond = 1 + ... circle = 3 + ... alias_for_square = 2 + ... + >>> Shape.square + <Shape.square: 2> + >>> Shape.alias_for_square + <Shape.square: 2> + >>> Shape(2) + <Shape.square: 2> + + + Allowing aliases is not always desirable. ``unique`` can be used to ensure + that none exist in a particular enumeration:: + + >>> from enum import unique + >>> @unique + ... class Mistake(Enum): + ... __order__ = 'one two three four' # only needed in 2.x + ... one = 1 + ... two = 2 + ... three = 3 + ... four = 3 + Traceback (most recent call last): + ... + ValueError: duplicate names found in <enum 'Mistake'>: four -> three + + Iterating over the members of an enum does not provide the aliases:: + + >>> list(Shape) + [<Shape.square: 2>, <Shape.diamond: 1>, <Shape.circle: 3>] + + The special attribute ``__members__`` is a dictionary mapping names to members. + It includes all names defined in the enumeration, including the aliases:: + + >>> for name, member in sorted(Shape.__members__.items()): + ... name, member + ... + ('alias_for_square', <Shape.square: 2>) + ('circle', <Shape.circle: 3>) + ('diamond', <Shape.diamond: 1>) + ('square', <Shape.square: 2>) + + The ``__members__`` attribute can be used for detailed programmatic access to + the enumeration members. For example, finding all the aliases:: + + >>> [name for name, member in Shape.__members__.items() if member.name != name] + ['alias_for_square'] + + Comparisons + ----------- + + Enumeration members are compared by identity:: + + >>> Color.red is Color.red + True + >>> Color.red is Color.blue + False + >>> Color.red is not Color.blue + True + + Ordered comparisons between enumeration values are *not* supported. Enum + members are not integers (but see `IntEnum`_ below):: + + >>> Color.red < Color.blue + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + TypeError: unorderable types: Color() < Color() + + .. warning:: + + In Python 2 *everything* is ordered, even though the ordering may not + make sense. If you want your enumerations to have a sensible ordering + check out the `OrderedEnum`_ recipe below. + + + Equality comparisons are defined though:: + + >>> Color.blue == Color.red + False + >>> Color.blue != Color.red + True + >>> Color.blue == Color.blue + True + + Comparisons against non-enumeration values will always compare not equal + (again, ``IntEnum`` was explicitly designed to behave differently, see + below):: + + >>> Color.blue == 2 + False + + + Allowed members and attributes of enumerations + ---------------------------------------------- + + The examples above use integers for enumeration values. Using integers is + short and handy (and provided by default by the `Functional API`_), but not + strictly enforced. In the vast majority of use-cases, one doesn't care what + the actual value of an enumeration is. But if the value *is* important, + enumerations can have arbitrary values. + + Enumerations are Python classes, and can have methods and special methods as + usual. If we have this enumeration:: + + >>> class Mood(Enum): + ... funky = 1 + ... happy = 3 + ... + ... def describe(self): + ... # self is the member here + ... return self.name, self.value + ... + ... def __str__(self): + ... return 'my custom str! {0}'.format(self.value) + ... + ... @classmethod + ... def favorite_mood(cls): + ... # cls here is the enumeration + ... return cls.happy + + Then:: + + >>> Mood.favorite_mood() + <Mood.happy: 3> + >>> Mood.happy.describe() + ('happy', 3) + >>> str(Mood.funky) + 'my custom str! 1' + + The rules for what is allowed are as follows: _sunder_ names (starting and + ending with a single underscore) are reserved by enum and cannot be used; + all other attributes defined within an enumeration will become members of this + enumeration, with the exception of *__dunder__* names and descriptors (methods + are also descriptors). + + Note: + + If your enumeration defines ``__new__`` and/or ``__init__`` then + whatever value(s) were given to the enum member will be passed into + those methods. See `Planet`_ for an example. + + + Restricted subclassing of enumerations + -------------------------------------- + + Subclassing an enumeration is allowed only if the enumeration does not define + any members. So this is forbidden:: + + >>> class MoreColor(Color): + ... pink = 17 + Traceback (most recent call last): + ... + TypeError: Cannot extend enumerations + + But this is allowed:: + + >>> class Foo(Enum): + ... def some_behavior(self): + ... pass + ... + >>> class Bar(Foo): + ... happy = 1 + ... sad = 2 + ... + + Allowing subclassing of enums that define members would lead to a violation of + some important invariants of types and instances. On the other hand, it makes + sense to allow sharing some common behavior between a group of enumerations. + (See `OrderedEnum`_ for an example.) + + + Pickling + -------- + + Enumerations can be pickled and unpickled:: + + >>> from enum.test_enum import Fruit + >>> from pickle import dumps, loads + >>> Fruit.tomato is loads(dumps(Fruit.tomato, 2)) + True + + The usual restrictions for pickling apply: picklable enums must be defined in + the top level of a module, since unpickling requires them to be importable + from that module. + + Note: + + With pickle protocol version 4 (introduced in Python 3.4) it is possible + to easily pickle enums nested in other classes. + + + + Functional API + -------------- + + The ``Enum`` class is callable, providing the following functional API:: + + >>> Animal = Enum('Animal', 'ant bee cat dog') + >>> Animal + <enum 'Animal'> + >>> Animal.ant + <Animal.ant: 1> + >>> Animal.ant.value + 1 + >>> list(Animal) + [<Animal.ant: 1>, <Animal.bee: 2>, <Animal.cat: 3>, <Animal.dog: 4>] + + The semantics of this API resemble ``namedtuple``. The first argument + of the call to ``Enum`` is the name of the enumeration. + + The second argument is the *source* of enumeration member names. It can be a + whitespace-separated string of names, a sequence of names, a sequence of + 2-tuples with key/value pairs, or a mapping (e.g. dictionary) of names to + values. The last two options enable assigning arbitrary values to + enumerations; the others auto-assign increasing integers starting with 1. A + new class derived from ``Enum`` is returned. In other words, the above + assignment to ``Animal`` is equivalent to:: + + >>> class Animals(Enum): + ... ant = 1 + ... bee = 2 + ... cat = 3 + ... dog = 4 + + Pickling enums created with the functional API can be tricky as frame stack + implementation details are used to try and figure out which module the + enumeration is being created in (e.g. it will fail if you use a utility + function in separate module, and also may not work on IronPython or Jython). + The solution is to specify the module name explicitly as follows:: + + >>> Animals = Enum('Animals', 'ant bee cat dog', module=__name__) + + Derived Enumerations + -------------------- + + IntEnum + ^^^^^^^ + + A variation of ``Enum`` is provided which is also a subclass of + ``int``. Members of an ``IntEnum`` can be compared to integers; + by extension, integer enumerations of different types can also be compared + to each other:: + + >>> from enum import IntEnum + >>> class Shape(IntEnum): + ... circle = 1 + ... square = 2 + ... + >>> class Request(IntEnum): + ... post = 1 + ... get = 2 + ... + >>> Shape == 1 + False + >>> Shape.circle == 1 + True + >>> Shape.circle == Request.post + True + + However, they still can't be compared to standard ``Enum`` enumerations:: + + >>> class Shape(IntEnum): + ... circle = 1 + ... square = 2 + ... + >>> class Color(Enum): + ... red = 1 + ... green = 2 + ... + >>> Shape.circle == Color.red + False + + ``IntEnum`` values behave like integers in other ways you'd expect:: + + >>> int(Shape.circle) + 1 + >>> ['a', 'b', 'c'][Shape.circle] + 'b' + >>> [i for i in range(Shape.square)] + [0, 1] + + For the vast majority of code, ``Enum`` is strongly recommended, + since ``IntEnum`` breaks some semantic promises of an enumeration (by + being comparable to integers, and thus by transitivity to other + unrelated enumerations). It should be used only in special cases where + there's no other choice; for example, when integer constants are + replaced with enumerations and backwards compatibility is required with code + that still expects integers. + + + Others + ^^^^^^ + + While ``IntEnum`` is part of the ``enum`` module, it would be very + simple to implement independently:: + + class IntEnum(int, Enum): + pass + + This demonstrates how similar derived enumerations can be defined; for example + a ``StrEnum`` that mixes in ``str`` instead of ``int``. + + Some rules: + + 1. When subclassing ``Enum``, mix-in types must appear before + ``Enum`` itself in the sequence of bases, as in the ``IntEnum`` + example above. + 2. While ``Enum`` can have members of any type, once you mix in an + additional type, all the members must have values of that type, e.g. + ``int`` above. This restriction does not apply to mix-ins which only + add methods and don't specify another data type such as ``int`` or + ``str``. + 3. When another data type is mixed in, the ``value`` attribute is *not the + same* as the enum member itself, although it is equivalant and will compare + equal. + 4. %-style formatting: ``%s`` and ``%r`` call ``Enum``'s ``__str__`` and + ``__repr__`` respectively; other codes (such as ``%i`` or ``%h`` for + IntEnum) treat the enum member as its mixed-in type. + + Note: Prior to Python 3.4 there is a bug in ``str``'s %-formatting: ``int`` + subclasses are printed as strings and not numbers when the ``%d``, ``%i``, + or ``%u`` codes are used. + 5. ``str.__format__`` (or ``format``) will use the mixed-in + type's ``__format__``. If the ``Enum``'s ``str`` or + ``repr`` is desired use the ``!s`` or ``!r`` ``str`` format codes. + + + Decorators + ---------- + + unique + ^^^^^^ + + A ``class`` decorator specifically for enumerations. It searches an + enumeration's ``__members__`` gathering any aliases it finds; if any are + found ``ValueError`` is raised with the details:: + + >>> @unique + ... class NoDupes(Enum): + ... first = 'one' + ... second = 'two' + ... third = 'two' + Traceback (most recent call last): + ... + ValueError: duplicate names found in <enum 'NoDupes'>: third -> second + + + Interesting examples + -------------------- + + While ``Enum`` and ``IntEnum`` are expected to cover the majority of + use-cases, they cannot cover them all. Here are recipes for some different + types of enumerations that can be used directly, or as examples for creating + one's own. + + + AutoNumber + ^^^^^^^^^^ + + Avoids having to specify the value for each enumeration member:: + + >>> class AutoNumber(Enum): + ... def __new__(cls): + ... value = len(cls.__members__) + 1 + ... obj = object.__new__(cls) + ... obj._value_ = value + ... return obj + ... + >>> class Color(AutoNumber): + ... __order__ = "red green blue" # only needed in 2.x + ... red = () + ... green = () + ... blue = () + ... + >>> Color.green.value == 2 + True + + Note: + + The `__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's `__new__` which is used after + class creation for lookup of existing members. Due to the way Enums are + supposed to behave, there is no way to customize Enum's `__new__`. + + + UniqueEnum + ^^^^^^^^^^ + + Raises an error if a duplicate member name is found instead of creating an + alias:: + + >>> class UniqueEnum(Enum): + ... def __init__(self, *args): + ... cls = self.__class__ + ... if any(self.value == e.value for e in cls): + ... a = self.name + ... e = cls(self.value).name + ... raise ValueError( + ... "aliases not allowed in UniqueEnum: %r --> %r" + ... % (a, e)) + ... + >>> class Color(UniqueEnum): + ... red = 1 + ... green = 2 + ... blue = 3 + ... grene = 2 + Traceback (most recent call last): + ... + ValueError: aliases not allowed in UniqueEnum: 'grene' --> 'green' + + + OrderedEnum + ^^^^^^^^^^^ + + An ordered enumeration that is not based on ``IntEnum`` and so maintains + the normal ``Enum`` invariants (such as not being comparable to other + enumerations):: + + >>> class OrderedEnum(Enum): + ... def __ge__(self, other): + ... if self.__class__ is other.__class__: + ... return self._value_ >= other._value_ + ... return NotImplemented + ... def __gt__(self, other): + ... if self.__class__ is other.__class__: + ... return self._value_ > other._value_ + ... return NotImplemented + ... def __le__(self, other): + ... if self.__class__ is other.__class__: + ... return self._value_ <= other._value_ + ... return NotImplemented + ... def __lt__(self, other): + ... if self.__class__ is other.__class__: + ... return self._value_ < other._value_ + ... return NotImplemented + ... + >>> class Grade(OrderedEnum): + ... __ordered__ = 'A B C D F' + ... A = 5 + ... B = 4 + ... C = 3 + ... D = 2 + ... F = 1 + ... + >>> Grade.C < Grade.A + True + + + Planet + ^^^^^^ + + If ``__new__`` or ``__init__`` is defined the value of the enum member + will be passed to those methods:: + + >>> class Planet(Enum): + ... MERCURY = (3.303e+23, 2.4397e6) + ... VENUS = (4.869e+24, 6.0518e6) + ... EARTH = (5.976e+24, 6.37814e6) + ... MARS = (6.421e+23, 3.3972e6) + ... JUPITER = (1.9e+27, 7.1492e7) + ... SATURN = (5.688e+26, 6.0268e7) + ... URANUS = (8.686e+25, 2.5559e7) + ... NEPTUNE = (1.024e+26, 2.4746e7) + ... def __init__(self, mass, radius): + ... self.mass = mass # in kilograms + ... self.radius = radius # in meters + ... @property + ... def surface_gravity(self): + ... # universal gravitational constant (m3 kg-1 s-2) + ... G = 6.67300E-11 + ... return G * self.mass / (self.radius * self.radius) + ... + >>> Planet.EARTH.value + (5.976e+24, 6378140.0) + >>> Planet.EARTH.surface_gravity + 9.802652743337129 + + + How are Enums different? + ------------------------ + + Enums have a custom metaclass that affects many aspects of both derived Enum + classes and their instances (members). + + + Enum Classes + ^^^^^^^^^^^^ + + The ``EnumMeta`` metaclass is responsible for providing the + ``__contains__``, ``__dir__``, ``__iter__`` and other methods that + allow one to do things with an ``Enum`` class that fail on a typical + class, such as ``list(Color)`` or ``some_var in Color``. ``EnumMeta`` is + responsible for ensuring that various other methods on the final ``Enum`` + class are correct (such as ``__new__``, ``__getnewargs__``, + ``__str__`` and ``__repr__``) + + + Enum Members (aka instances) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + The most interesting thing about Enum members is that they are singletons. + ``EnumMeta`` creates them all while it is creating the ``Enum`` + class itself, and then puts a custom ``__new__`` in place to ensure + that no new ones are ever instantiated by returning only the existing + member instances. + + + Finer Points + ^^^^^^^^^^^^ + + Enum members are instances of an Enum class, and even though they are + accessible as ``EnumClass.member``, they are not accessible directly from + the member:: + + >>> Color.red + <Color.red: 1> + >>> Color.red.blue + Traceback (most recent call last): + ... + AttributeError: 'Color' object has no attribute 'blue' + + Likewise, ``__members__`` is only available on the class. + + In Python 3.x ``__members__`` is always an ``OrderedDict``, with the order being + the definition order. In Python 2.7 ``__members__`` is an ``OrderedDict`` if + ``__order__`` was specified, and a plain ``dict`` otherwise. In all other Python + 2.x versions ``__members__`` is a plain ``dict`` even if ``__order__`` was specified + as the ``OrderedDict`` type didn't exist yet. + + If you give your ``Enum`` subclass extra methods, like the `Planet`_ + class above, those methods will show up in a `dir` of the member, + but not of the class:: + + >>> dir(Planet) + ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', + 'VENUS', '__class__', '__doc__', '__members__', '__module__'] + >>> dir(Planet.EARTH) + ['__class__', '__doc__', '__module__', 'name', 'surface_gravity', 'value'] + + A ``__new__`` method will only be used for the creation of the + ``Enum`` members -- after that it is replaced. This means if you wish to + change how ``Enum`` members are looked up you either have to write a + helper function or a ``classmethod``. + +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development +Classifier: Programming Language :: Python :: 2.4 +Classifier: Programming Language :: Python :: 2.5 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Provides: enum diff --git a/scripts/external_libs/enum34-1.0.4/enum/LICENSE b/scripts/external_libs/enum34-1.0.4/enum/LICENSE new file mode 100644 index 00000000..9003b885 --- /dev/null +++ b/scripts/external_libs/enum34-1.0.4/enum/LICENSE @@ -0,0 +1,32 @@ +Copyright (c) 2013, Ethan Furman. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + + Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + + Neither the name Ethan Furman nor the names of any + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/scripts/external_libs/enum34-1.0.4/enum/README b/scripts/external_libs/enum34-1.0.4/enum/README new file mode 100644 index 00000000..511af984 --- /dev/null +++ b/scripts/external_libs/enum34-1.0.4/enum/README @@ -0,0 +1,2 @@ +enum34 is the new Python stdlib enum module available in Python 3.4 +backported for previous versions of Python from 2.4 to 3.3. diff --git a/scripts/external_libs/enum34-1.0.4/enum/__init__.py b/scripts/external_libs/enum34-1.0.4/enum/__init__.py new file mode 100644 index 00000000..6a327a8a --- /dev/null +++ b/scripts/external_libs/enum34-1.0.4/enum/__init__.py @@ -0,0 +1,790 @@ +"""Python Enumerations""" + +import sys as _sys + +__all__ = ['Enum', 'IntEnum', 'unique'] + +version = 1, 0, 4 + +pyver = float('%s.%s' % _sys.version_info[:2]) + +try: + any +except NameError: + def any(iterable): + for element in iterable: + if element: + return True + return False + +try: + from collections import OrderedDict +except ImportError: + OrderedDict = None + +try: + basestring +except NameError: + # In Python 2 basestring is the ancestor of both str and unicode + # in Python 3 it's just str, but was missing in 3.1 + basestring = str + +try: + unicode +except NameError: + # In Python 3 unicode no longer exists (it's just str) + unicode = str + +class _RouteClassAttributeToGetattr(object): + """Route attribute access on a class to __getattr__. + + This is a descriptor, used to define attributes that act differently when + accessed through an instance and through a class. Instance access remains + normal, but access to an attribute through a class will be routed to the + class's __getattr__ method; this is done by raising AttributeError. + + """ + def __init__(self, fget=None): + self.fget = fget + + def __get__(self, instance, ownerclass=None): + if instance is None: + raise AttributeError() + return self.fget(instance) + + def __set__(self, instance, value): + raise AttributeError("can't set attribute") + + def __delete__(self, instance): + raise AttributeError("can't delete attribute") + + +def _is_descriptor(obj): + """Returns True if obj is a descriptor, False otherwise.""" + return ( + hasattr(obj, '__get__') or + hasattr(obj, '__set__') or + hasattr(obj, '__delete__')) + + +def _is_dunder(name): + """Returns True if a __dunder__ name, False otherwise.""" + return (name[:2] == name[-2:] == '__' and + name[2:3] != '_' and + name[-3:-2] != '_' and + len(name) > 4) + + +def _is_sunder(name): + """Returns True if a _sunder_ name, False otherwise.""" + return (name[0] == name[-1] == '_' and + name[1:2] != '_' and + name[-2:-1] != '_' and + len(name) > 2) + + +def _make_class_unpicklable(cls): + """Make the given class un-picklable.""" + def _break_on_call_reduce(self, protocol=None): + raise TypeError('%r cannot be pickled' % self) + cls.__reduce_ex__ = _break_on_call_reduce + cls.__module__ = '<unknown>' + + +class _EnumDict(dict): + """Track enum member order and ensure member names are not reused. + + EnumMeta will use the names found in self._member_names as the + enumeration member names. + + """ + def __init__(self): + super(_EnumDict, self).__init__() + self._member_names = [] + + def __setitem__(self, key, value): + """Changes anything not dundered or not a descriptor. + + If a descriptor is added with the same name as an enum member, the name + is removed from _member_names (this may leave a hole in the numerical + sequence of values). + + If an enum member name is used twice, an error is raised; duplicate + values are not checked for. + + Single underscore (sunder) names are reserved. + + Note: in 3.x __order__ is simply discarded as a not necessary piece + leftover from 2.x + + """ + if pyver >= 3.0 and key == '__order__': + return + if _is_sunder(key): + raise ValueError('_names_ are reserved for future Enum use') + elif _is_dunder(key): + pass + elif key in self._member_names: + # descriptor overwriting an enum? + raise TypeError('Attempted to reuse key: %r' % key) + elif not _is_descriptor(value): + if key in self: + # enum overwriting a descriptor? + raise TypeError('Key already defined as: %r' % self[key]) + self._member_names.append(key) + super(_EnumDict, self).__setitem__(key, value) + + +# Dummy value for Enum as EnumMeta explicity checks for it, but of course until +# EnumMeta finishes running the first time the Enum class doesn't exist. This +# is also why there are checks in EnumMeta like `if Enum is not None` +Enum = None + + +class EnumMeta(type): + """Metaclass for Enum""" + @classmethod + def __prepare__(metacls, cls, bases): + return _EnumDict() + + def __new__(metacls, cls, bases, classdict): + # an Enum class is final once enumeration items have been defined; it + # cannot be mixed with other types (int, float, etc.) if it has an + # inherited __new__ unless a new __new__ is defined (or the resulting + # class will fail). + if type(classdict) is dict: + original_dict = classdict + classdict = _EnumDict() + for k, v in original_dict.items(): + classdict[k] = v + + member_type, first_enum = metacls._get_mixins_(bases) + __new__, save_new, use_args = metacls._find_new_(classdict, member_type, + first_enum) + # save enum items into separate mapping so they don't get baked into + # the new class + members = dict((k, classdict[k]) for k in classdict._member_names) + for name in classdict._member_names: + del classdict[name] + + # py2 support for definition order + __order__ = classdict.get('__order__') + if __order__ is None: + if pyver < 3.0: + try: + __order__ = [name for (name, value) in sorted(members.items(), key=lambda item: item[1])] + except TypeError: + __order__ = [name for name in sorted(members.keys())] + else: + __order__ = classdict._member_names + else: + del classdict['__order__'] + if pyver < 3.0: + __order__ = __order__.replace(',', ' ').split() + aliases = [name for name in members if name not in __order__] + __order__ += aliases + + # check for illegal enum names (any others?) + invalid_names = set(members) & set(['mro']) + if invalid_names: + raise ValueError('Invalid enum member name(s): %s' % ( + ', '.join(invalid_names), )) + + # create our new Enum type + enum_class = super(EnumMeta, metacls).__new__(metacls, cls, bases, classdict) + enum_class._member_names_ = [] # names in random order + if OrderedDict is not None: + enum_class._member_map_ = OrderedDict() + else: + enum_class._member_map_ = {} # name->value map + enum_class._member_type_ = member_type + + # Reverse value->name map for hashable values. + enum_class._value2member_map_ = {} + + # instantiate them, checking for duplicates as we go + # we instantiate first instead of checking for duplicates first in case + # a custom __new__ is doing something funky with the values -- such as + # auto-numbering ;) + if __new__ is None: + __new__ = enum_class.__new__ + for member_name in __order__: + value = members[member_name] + if not isinstance(value, tuple): + args = (value, ) + else: + args = value + if member_type is tuple: # special case for tuple enums + args = (args, ) # wrap it one more time + if not use_args or not args: + enum_member = __new__(enum_class) + if not hasattr(enum_member, '_value_'): + enum_member._value_ = value + else: + enum_member = __new__(enum_class, *args) + if not hasattr(enum_member, '_value_'): + enum_member._value_ = member_type(*args) + value = enum_member._value_ + enum_member._name_ = member_name + enum_member.__objclass__ = enum_class + enum_member.__init__(*args) + # If another member with the same value was already defined, the + # new member becomes an alias to the existing one. + for name, canonical_member in enum_class._member_map_.items(): + if canonical_member.value == enum_member._value_: + enum_member = canonical_member + break + else: + # Aliases don't appear in member names (only in __members__). + enum_class._member_names_.append(member_name) + enum_class._member_map_[member_name] = enum_member + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_[value] = enum_member + except TypeError: + pass + + + # If a custom type is mixed into the Enum, and it does not know how + # to pickle itself, pickle.dumps will succeed but pickle.loads will + # fail. Rather than have the error show up later and possibly far + # from the source, sabotage the pickle protocol for this class so + # that pickle.dumps also fails. + # + # However, if the new class implements its own __reduce_ex__, do not + # sabotage -- it's on them to make sure it works correctly. We use + # __reduce_ex__ instead of any of the others as it is preferred by + # pickle over __reduce__, and it handles all pickle protocols. + unpicklable = False + if '__reduce_ex__' not in classdict: + if member_type is not object: + methods = ('__getnewargs_ex__', '__getnewargs__', + '__reduce_ex__', '__reduce__') + if not any(m in member_type.__dict__ for m in methods): + _make_class_unpicklable(enum_class) + unpicklable = True + + + # double check that repr and friends are not the mixin's or various + # things break (such as pickle) + for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'): + class_method = getattr(enum_class, name) + obj_method = getattr(member_type, name, None) + enum_method = getattr(first_enum, name, None) + if name not in classdict and class_method is not enum_method: + if name == '__reduce_ex__' and unpicklable: + continue + setattr(enum_class, name, enum_method) + + # method resolution and int's are not playing nice + # Python's less than 2.6 use __cmp__ + + if pyver < 2.6: + + if issubclass(enum_class, int): + setattr(enum_class, '__cmp__', getattr(int, '__cmp__')) + + elif pyver < 3.0: + + if issubclass(enum_class, int): + for method in ( + '__le__', + '__lt__', + '__gt__', + '__ge__', + '__eq__', + '__ne__', + '__hash__', + ): + setattr(enum_class, method, getattr(int, method)) + + # replace any other __new__ with our own (as long as Enum is not None, + # anyway) -- again, this is to support pickle + if Enum is not None: + # if the user defined their own __new__, save it before it gets + # clobbered in case they subclass later + if save_new: + setattr(enum_class, '__member_new__', enum_class.__dict__['__new__']) + setattr(enum_class, '__new__', Enum.__dict__['__new__']) + return enum_class + + def __call__(cls, value, names=None, module=None, type=None): + """Either returns an existing member, or creates a new enum class. + + This method is used both when an enum class is given a value to match + to an enumeration member (i.e. Color(3)) and for the functional API + (i.e. Color = Enum('Color', names='red green blue')). + + When used for the functional API: `module`, if set, will be stored in + the new class' __module__ attribute; `type`, if set, will be mixed in + as the first base class. + + Note: if `module` is not set this routine will attempt to discover the + calling module by walking the frame stack; if this is unsuccessful + the resulting class will not be pickleable. + + """ + if names is None: # simple value lookup + return cls.__new__(cls, value) + # otherwise, functional API: we're creating a new Enum type + return cls._create_(value, names, module=module, type=type) + + def __contains__(cls, member): + return isinstance(member, cls) and member.name in cls._member_map_ + + def __delattr__(cls, attr): + # nicer error message when someone tries to delete an attribute + # (see issue19025). + if attr in cls._member_map_: + raise AttributeError( + "%s: cannot delete Enum member." % cls.__name__) + super(EnumMeta, cls).__delattr__(attr) + + def __dir__(self): + return (['__class__', '__doc__', '__members__', '__module__'] + + self._member_names_) + + @property + def __members__(cls): + """Returns a mapping of member name->value. + + This mapping lists all enum members, including aliases. Note that this + is a copy of the internal mapping. + + """ + return cls._member_map_.copy() + + def __getattr__(cls, name): + """Return the enum member matching `name` + + We use __getattr__ instead of descriptors or inserting into the enum + class' __dict__ in order to support `name` and `value` being both + properties for enum members (which live in the class' __dict__) and + enum members themselves. + + """ + if _is_dunder(name): + raise AttributeError(name) + try: + return cls._member_map_[name] + except KeyError: + raise AttributeError(name) + + def __getitem__(cls, name): + return cls._member_map_[name] + + def __iter__(cls): + return (cls._member_map_[name] for name in cls._member_names_) + + def __reversed__(cls): + return (cls._member_map_[name] for name in reversed(cls._member_names_)) + + def __len__(cls): + return len(cls._member_names_) + + def __repr__(cls): + return "<enum %r>" % cls.__name__ + + def __setattr__(cls, name, value): + """Block attempts to reassign Enum members. + + A simple assignment to the class namespace only changes one of the + several possible ways to get an Enum member from the Enum class, + resulting in an inconsistent Enumeration. + + """ + member_map = cls.__dict__.get('_member_map_', {}) + if name in member_map: + raise AttributeError('Cannot reassign members.') + super(EnumMeta, cls).__setattr__(name, value) + + def _create_(cls, class_name, names=None, module=None, type=None): + """Convenience method to create a new Enum class. + + `names` can be: + + * A string containing member names, separated either with spaces or + commas. Values are auto-numbered from 1. + * An iterable of member names. Values are auto-numbered from 1. + * An iterable of (member name, value) pairs. + * A mapping of member name -> value. + + """ + if pyver < 3.0: + # if class_name is unicode, attempt a conversion to ASCII + if isinstance(class_name, unicode): + try: + class_name = class_name.encode('ascii') + except UnicodeEncodeError: + raise TypeError('%r is not representable in ASCII' % class_name) + metacls = cls.__class__ + if type is None: + bases = (cls, ) + else: + bases = (type, cls) + classdict = metacls.__prepare__(class_name, bases) + __order__ = [] + + # special processing needed for names? + if isinstance(names, basestring): + names = names.replace(',', ' ').split() + if isinstance(names, (tuple, list)) and isinstance(names[0], basestring): + names = [(e, i+1) for (i, e) in enumerate(names)] + + # Here, names is either an iterable of (name, value) or a mapping. + for item in names: + if isinstance(item, basestring): + member_name, member_value = item, names[item] + else: + member_name, member_value = item + classdict[member_name] = member_value + __order__.append(member_name) + # only set __order__ in classdict if name/value was not from a mapping + if not isinstance(item, basestring): + classdict['__order__'] = ' '.join(__order__) + enum_class = metacls.__new__(metacls, class_name, bases, classdict) + + # TODO: replace the frame hack if a blessed way to know the calling + # module is ever developed + if module is None: + try: + module = _sys._getframe(2).f_globals['__name__'] + except (AttributeError, ValueError): + pass + if module is None: + _make_class_unpicklable(enum_class) + else: + enum_class.__module__ = module + + return enum_class + + @staticmethod + def _get_mixins_(bases): + """Returns the type for creating enum members, and the first inherited + enum class. + + bases: the tuple of bases that was given to __new__ + + """ + if not bases or Enum is None: + return object, Enum + + + # double check that we are not subclassing a class with existing + # enumeration members; while we're at it, see if any other data + # type has been mixed in so we can use the correct __new__ + member_type = first_enum = None + for base in bases: + if (base is not Enum and + issubclass(base, Enum) and + base._member_names_): + raise TypeError("Cannot extend enumerations") + # base is now the last base in bases + if not issubclass(base, Enum): + raise TypeError("new enumerations must be created as " + "`ClassName([mixin_type,] enum_type)`") + + # get correct mix-in type (either mix-in type of Enum subclass, or + # first base if last base is Enum) + if not issubclass(bases[0], Enum): + member_type = bases[0] # first data type + first_enum = bases[-1] # enum type + else: + for base in bases[0].__mro__: + # most common: (IntEnum, int, Enum, object) + # possible: (<Enum 'AutoIntEnum'>, <Enum 'IntEnum'>, + # <class 'int'>, <Enum 'Enum'>, + # <class 'object'>) + if issubclass(base, Enum): + if first_enum is None: + first_enum = base + else: + if member_type is None: + member_type = base + + return member_type, first_enum + + if pyver < 3.0: + @staticmethod + def _find_new_(classdict, member_type, first_enum): + """Returns the __new__ to be used for creating the enum members. + + classdict: the class dictionary given to __new__ + member_type: the data type whose __new__ will be used by default + first_enum: enumeration to check for an overriding __new__ + + """ + # now find the correct __new__, checking to see of one was defined + # by the user; also check earlier enum classes in case a __new__ was + # saved as __member_new__ + __new__ = classdict.get('__new__', None) + if __new__: + return None, True, True # __new__, save_new, use_args + + N__new__ = getattr(None, '__new__') + O__new__ = getattr(object, '__new__') + if Enum is None: + E__new__ = N__new__ + else: + E__new__ = Enum.__dict__['__new__'] + # check all possibles for __member_new__ before falling back to + # __new__ + for method in ('__member_new__', '__new__'): + for possible in (member_type, first_enum): + try: + target = possible.__dict__[method] + except (AttributeError, KeyError): + target = getattr(possible, method, None) + if target not in [ + None, + N__new__, + O__new__, + E__new__, + ]: + if method == '__member_new__': + classdict['__new__'] = target + return None, False, True + if isinstance(target, staticmethod): + target = target.__get__(member_type) + __new__ = target + break + if __new__ is not None: + break + else: + __new__ = object.__new__ + + # if a non-object.__new__ is used then whatever value/tuple was + # assigned to the enum member name will be passed to __new__ and to the + # new enum member's __init__ + if __new__ is object.__new__: + use_args = False + else: + use_args = True + + return __new__, False, use_args + else: + @staticmethod + def _find_new_(classdict, member_type, first_enum): + """Returns the __new__ to be used for creating the enum members. + + classdict: the class dictionary given to __new__ + member_type: the data type whose __new__ will be used by default + first_enum: enumeration to check for an overriding __new__ + + """ + # now find the correct __new__, checking to see of one was defined + # by the user; also check earlier enum classes in case a __new__ was + # saved as __member_new__ + __new__ = classdict.get('__new__', None) + + # should __new__ be saved as __member_new__ later? + save_new = __new__ is not None + + if __new__ is None: + # check all possibles for __member_new__ before falling back to + # __new__ + for method in ('__member_new__', '__new__'): + for possible in (member_type, first_enum): + target = getattr(possible, method, None) + if target not in ( + None, + None.__new__, + object.__new__, + Enum.__new__, + ): + __new__ = target + break + if __new__ is not None: + break + else: + __new__ = object.__new__ + + # if a non-object.__new__ is used then whatever value/tuple was + # assigned to the enum member name will be passed to __new__ and to the + # new enum member's __init__ + if __new__ is object.__new__: + use_args = False + else: + use_args = True + + return __new__, save_new, use_args + + +######################################################## +# In order to support Python 2 and 3 with a single +# codebase we have to create the Enum methods separately +# and then use the `type(name, bases, dict)` method to +# create the class. +######################################################## +temp_enum_dict = {} +temp_enum_dict['__doc__'] = "Generic enumeration.\n\n Derive from this class to define new enumerations.\n\n" + +def __new__(cls, value): + # all enum instances are actually created during class construction + # without calling this method; this method is called by the metaclass' + # __call__ (i.e. Color(3) ), and by pickle + if type(value) is cls: + # For lookups like Color(Color.red) + value = value.value + #return value + # by-value search for a matching enum member + # see if it's in the reverse mapping (for hashable values) + try: + if value in cls._value2member_map_: + return cls._value2member_map_[value] + except TypeError: + # not there, now do long search -- O(n) behavior + for member in cls._member_map_.values(): + if member.value == value: + return member + raise ValueError("%s is not a valid %s" % (value, cls.__name__)) +temp_enum_dict['__new__'] = __new__ +del __new__ + +def __repr__(self): + return "<%s.%s: %r>" % ( + self.__class__.__name__, self._name_, self._value_) +temp_enum_dict['__repr__'] = __repr__ +del __repr__ + +def __str__(self): + return "%s.%s" % (self.__class__.__name__, self._name_) +temp_enum_dict['__str__'] = __str__ +del __str__ + +def __dir__(self): + added_behavior = [ + m + for cls in self.__class__.mro() + for m in cls.__dict__ + if m[0] != '_' + ] + return (['__class__', '__doc__', '__module__', ] + added_behavior) +temp_enum_dict['__dir__'] = __dir__ +del __dir__ + +def __format__(self, format_spec): + # mixed-in Enums should use the mixed-in type's __format__, otherwise + # we can get strange results with the Enum name showing up instead of + # the value + + # pure Enum branch + if self._member_type_ is object: + cls = str + val = str(self) + # mix-in branch + else: + cls = self._member_type_ + val = self.value + return cls.__format__(val, format_spec) +temp_enum_dict['__format__'] = __format__ +del __format__ + + +#################################### +# Python's less than 2.6 use __cmp__ + +if pyver < 2.6: + + def __cmp__(self, other): + if type(other) is self.__class__: + if self is other: + return 0 + return -1 + return NotImplemented + raise TypeError("unorderable types: %s() and %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__cmp__'] = __cmp__ + del __cmp__ + +else: + + def __le__(self, other): + raise TypeError("unorderable types: %s() <= %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__le__'] = __le__ + del __le__ + + def __lt__(self, other): + raise TypeError("unorderable types: %s() < %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__lt__'] = __lt__ + del __lt__ + + def __ge__(self, other): + raise TypeError("unorderable types: %s() >= %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__ge__'] = __ge__ + del __ge__ + + def __gt__(self, other): + raise TypeError("unorderable types: %s() > %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__gt__'] = __gt__ + del __gt__ + + +def __eq__(self, other): + if type(other) is self.__class__: + return self is other + return NotImplemented +temp_enum_dict['__eq__'] = __eq__ +del __eq__ + +def __ne__(self, other): + if type(other) is self.__class__: + return self is not other + return NotImplemented +temp_enum_dict['__ne__'] = __ne__ +del __ne__ + +def __hash__(self): + return hash(self._name_) +temp_enum_dict['__hash__'] = __hash__ +del __hash__ + +def __reduce_ex__(self, proto): + return self.__class__, (self._value_, ) +temp_enum_dict['__reduce_ex__'] = __reduce_ex__ +del __reduce_ex__ + +# _RouteClassAttributeToGetattr is used to provide access to the `name` +# and `value` properties of enum members while keeping some measure of +# protection from modification, while still allowing for an enumeration +# to have members named `name` and `value`. This works because enumeration +# members are not set directly on the enum class -- __getattr__ is +# used to look them up. + +@_RouteClassAttributeToGetattr +def name(self): + return self._name_ +temp_enum_dict['name'] = name +del name + +@_RouteClassAttributeToGetattr +def value(self): + return self._value_ +temp_enum_dict['value'] = value +del value + +Enum = EnumMeta('Enum', (object, ), temp_enum_dict) +del temp_enum_dict + +# Enum has now been created +########################### + +class IntEnum(int, Enum): + """Enum where members are also (and must be) ints""" + + +def unique(enumeration): + """Class decorator that ensures only unique members exist in an enumeration.""" + duplicates = [] + for name, member in enumeration.__members__.items(): + if name != member.name: + duplicates.append((name, member.name)) + if duplicates: + duplicate_names = ', '.join( + ["%s -> %s" % (alias, name) for (alias, name) in duplicates] + ) + raise ValueError('duplicate names found in %r: %s' % + (enumeration, duplicate_names) + ) + return enumeration diff --git a/scripts/external_libs/enum34-1.0.4/enum/doc/enum.rst b/scripts/external_libs/enum34-1.0.4/enum/doc/enum.rst new file mode 100644 index 00000000..0d429bfc --- /dev/null +++ b/scripts/external_libs/enum34-1.0.4/enum/doc/enum.rst @@ -0,0 +1,725 @@ +``enum`` --- support for enumerations +======================================== + +.. :synopsis: enumerations are sets of symbolic names bound to unique, constant + values. +.. :moduleauthor:: Ethan Furman <ethan@stoneleaf.us> +.. :sectionauthor:: Barry Warsaw <barry@python.org>, +.. :sectionauthor:: Eli Bendersky <eliben@gmail.com>, +.. :sectionauthor:: Ethan Furman <ethan@stoneleaf.us> + +---------------- + +An enumeration is a set of symbolic names (members) bound to unique, constant +values. Within an enumeration, the members can be compared by identity, and +the enumeration itself can be iterated over. + + +Module Contents +--------------- + +This module defines two enumeration classes that can be used to define unique +sets of names and values: ``Enum`` and ``IntEnum``. It also defines +one decorator, ``unique``. + +``Enum`` + +Base class for creating enumerated constants. See section `Functional API`_ +for an alternate construction syntax. + +``IntEnum`` + +Base class for creating enumerated constants that are also subclasses of ``int``. + +``unique`` + +Enum class decorator that ensures only one name is bound to any one value. + + +Creating an Enum +---------------- + +Enumerations are created using the ``class`` syntax, which makes them +easy to read and write. An alternative creation method is described in +`Functional API`_. To define an enumeration, subclass ``Enum`` as +follows:: + + >>> from enum import Enum + >>> class Color(Enum): + ... red = 1 + ... green = 2 + ... blue = 3 + +Note: Nomenclature + + - The class ``Color`` is an *enumeration* (or *enum*) + - The attributes ``Color.red``, ``Color.green``, etc., are + *enumeration members* (or *enum members*). + - The enum members have *names* and *values* (the name of + ``Color.red`` is ``red``, the value of ``Color.blue`` is + ``3``, etc.) + +Note: + + Even though we use the ``class`` syntax to create Enums, Enums + are not normal Python classes. See `How are Enums different?`_ for + more details. + +Enumeration members have human readable string representations:: + + >>> print(Color.red) + Color.red + +...while their ``repr`` has more information:: + + >>> print(repr(Color.red)) + <Color.red: 1> + +The *type* of an enumeration member is the enumeration it belongs to:: + + >>> type(Color.red) + <enum 'Color'> + >>> isinstance(Color.green, Color) + True + >>> + +Enum members also have a property that contains just their item name:: + + >>> print(Color.red.name) + red + +Enumerations support iteration. In Python 3.x definition order is used; in +Python 2.x the definition order is not available, but class attribute +``__order__`` is supported; otherwise, value order is used:: + + >>> class Shake(Enum): + ... __order__ = 'vanilla chocolate cookies mint' # only needed in 2.x + ... vanilla = 7 + ... chocolate = 4 + ... cookies = 9 + ... mint = 3 + ... + >>> for shake in Shake: + ... print(shake) + ... + Shake.vanilla + Shake.chocolate + Shake.cookies + Shake.mint + +The ``__order__`` attribute is always removed, and in 3.x it is also ignored +(order is definition order); however, in the stdlib version it will be ignored +but not removed. + +Enumeration members are hashable, so they can be used in dictionaries and sets:: + + >>> apples = {} + >>> apples[Color.red] = 'red delicious' + >>> apples[Color.green] = 'granny smith' + >>> apples == {Color.red: 'red delicious', Color.green: 'granny smith'} + True + + +Programmatic access to enumeration members and their attributes +--------------------------------------------------------------- + +Sometimes it's useful to access members in enumerations programmatically (i.e. +situations where ``Color.red`` won't do because the exact color is not known +at program-writing time). ``Enum`` allows such access:: + + >>> Color(1) + <Color.red: 1> + >>> Color(3) + <Color.blue: 3> + +If you want to access enum members by *name*, use item access:: + + >>> Color['red'] + <Color.red: 1> + >>> Color['green'] + <Color.green: 2> + +If have an enum member and need its ``name`` or ``value``:: + + >>> member = Color.red + >>> member.name + 'red' + >>> member.value + 1 + + +Duplicating enum members and values +----------------------------------- + +Having two enum members (or any other attribute) with the same name is invalid; +in Python 3.x this would raise an error, but in Python 2.x the second member +simply overwrites the first:: + + >>> # python 2.x + >>> class Shape(Enum): + ... square = 2 + ... square = 3 + ... + >>> Shape.square + <Shape.square: 3> + + >>> # python 3.x + >>> class Shape(Enum): + ... square = 2 + ... square = 3 + Traceback (most recent call last): + ... + TypeError: Attempted to reuse key: 'square' + +However, two enum members are allowed to have the same value. Given two members +A and B with the same value (and A defined first), B is an alias to A. By-value +lookup of the value of A and B will return A. By-name lookup of B will also +return A:: + + >>> class Shape(Enum): + ... __order__ = 'square diamond circle alias_for_square' # only needed in 2.x + ... square = 2 + ... diamond = 1 + ... circle = 3 + ... alias_for_square = 2 + ... + >>> Shape.square + <Shape.square: 2> + >>> Shape.alias_for_square + <Shape.square: 2> + >>> Shape(2) + <Shape.square: 2> + + +Allowing aliases is not always desirable. ``unique`` can be used to ensure +that none exist in a particular enumeration:: + + >>> from enum import unique + >>> @unique + ... class Mistake(Enum): + ... __order__ = 'one two three four' # only needed in 2.x + ... one = 1 + ... two = 2 + ... three = 3 + ... four = 3 + Traceback (most recent call last): + ... + ValueError: duplicate names found in <enum 'Mistake'>: four -> three + +Iterating over the members of an enum does not provide the aliases:: + + >>> list(Shape) + [<Shape.square: 2>, <Shape.diamond: 1>, <Shape.circle: 3>] + +The special attribute ``__members__`` is a dictionary mapping names to members. +It includes all names defined in the enumeration, including the aliases:: + + >>> for name, member in sorted(Shape.__members__.items()): + ... name, member + ... + ('alias_for_square', <Shape.square: 2>) + ('circle', <Shape.circle: 3>) + ('diamond', <Shape.diamond: 1>) + ('square', <Shape.square: 2>) + +The ``__members__`` attribute can be used for detailed programmatic access to +the enumeration members. For example, finding all the aliases:: + + >>> [name for name, member in Shape.__members__.items() if member.name != name] + ['alias_for_square'] + +Comparisons +----------- + +Enumeration members are compared by identity:: + + >>> Color.red is Color.red + True + >>> Color.red is Color.blue + False + >>> Color.red is not Color.blue + True + +Ordered comparisons between enumeration values are *not* supported. Enum +members are not integers (but see `IntEnum`_ below):: + + >>> Color.red < Color.blue + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + TypeError: unorderable types: Color() < Color() + +.. warning:: + + In Python 2 *everything* is ordered, even though the ordering may not + make sense. If you want your enumerations to have a sensible ordering + check out the `OrderedEnum`_ recipe below. + + +Equality comparisons are defined though:: + + >>> Color.blue == Color.red + False + >>> Color.blue != Color.red + True + >>> Color.blue == Color.blue + True + +Comparisons against non-enumeration values will always compare not equal +(again, ``IntEnum`` was explicitly designed to behave differently, see +below):: + + >>> Color.blue == 2 + False + + +Allowed members and attributes of enumerations +---------------------------------------------- + +The examples above use integers for enumeration values. Using integers is +short and handy (and provided by default by the `Functional API`_), but not +strictly enforced. In the vast majority of use-cases, one doesn't care what +the actual value of an enumeration is. But if the value *is* important, +enumerations can have arbitrary values. + +Enumerations are Python classes, and can have methods and special methods as +usual. If we have this enumeration:: + + >>> class Mood(Enum): + ... funky = 1 + ... happy = 3 + ... + ... def describe(self): + ... # self is the member here + ... return self.name, self.value + ... + ... def __str__(self): + ... return 'my custom str! {0}'.format(self.value) + ... + ... @classmethod + ... def favorite_mood(cls): + ... # cls here is the enumeration + ... return cls.happy + +Then:: + + >>> Mood.favorite_mood() + <Mood.happy: 3> + >>> Mood.happy.describe() + ('happy', 3) + >>> str(Mood.funky) + 'my custom str! 1' + +The rules for what is allowed are as follows: _sunder_ names (starting and +ending with a single underscore) are reserved by enum and cannot be used; +all other attributes defined within an enumeration will become members of this +enumeration, with the exception of *__dunder__* names and descriptors (methods +are also descriptors). + +Note: + + If your enumeration defines ``__new__`` and/or ``__init__`` then + whatever value(s) were given to the enum member will be passed into + those methods. See `Planet`_ for an example. + + +Restricted subclassing of enumerations +-------------------------------------- + +Subclassing an enumeration is allowed only if the enumeration does not define +any members. So this is forbidden:: + + >>> class MoreColor(Color): + ... pink = 17 + Traceback (most recent call last): + ... + TypeError: Cannot extend enumerations + +But this is allowed:: + + >>> class Foo(Enum): + ... def some_behavior(self): + ... pass + ... + >>> class Bar(Foo): + ... happy = 1 + ... sad = 2 + ... + +Allowing subclassing of enums that define members would lead to a violation of +some important invariants of types and instances. On the other hand, it makes +sense to allow sharing some common behavior between a group of enumerations. +(See `OrderedEnum`_ for an example.) + + +Pickling +-------- + +Enumerations can be pickled and unpickled:: + + >>> from enum.test_enum import Fruit + >>> from pickle import dumps, loads + >>> Fruit.tomato is loads(dumps(Fruit.tomato, 2)) + True + +The usual restrictions for pickling apply: picklable enums must be defined in +the top level of a module, since unpickling requires them to be importable +from that module. + +Note: + + With pickle protocol version 4 (introduced in Python 3.4) it is possible + to easily pickle enums nested in other classes. + + + +Functional API +-------------- + +The ``Enum`` class is callable, providing the following functional API:: + + >>> Animal = Enum('Animal', 'ant bee cat dog') + >>> Animal + <enum 'Animal'> + >>> Animal.ant + <Animal.ant: 1> + >>> Animal.ant.value + 1 + >>> list(Animal) + [<Animal.ant: 1>, <Animal.bee: 2>, <Animal.cat: 3>, <Animal.dog: 4>] + +The semantics of this API resemble ``namedtuple``. The first argument +of the call to ``Enum`` is the name of the enumeration. + +The second argument is the *source* of enumeration member names. It can be a +whitespace-separated string of names, a sequence of names, a sequence of +2-tuples with key/value pairs, or a mapping (e.g. dictionary) of names to +values. The last two options enable assigning arbitrary values to +enumerations; the others auto-assign increasing integers starting with 1. A +new class derived from ``Enum`` is returned. In other words, the above +assignment to ``Animal`` is equivalent to:: + + >>> class Animals(Enum): + ... ant = 1 + ... bee = 2 + ... cat = 3 + ... dog = 4 + +Pickling enums created with the functional API can be tricky as frame stack +implementation details are used to try and figure out which module the +enumeration is being created in (e.g. it will fail if you use a utility +function in separate module, and also may not work on IronPython or Jython). +The solution is to specify the module name explicitly as follows:: + + >>> Animals = Enum('Animals', 'ant bee cat dog', module=__name__) + +Derived Enumerations +-------------------- + +IntEnum +^^^^^^^ + +A variation of ``Enum`` is provided which is also a subclass of +``int``. Members of an ``IntEnum`` can be compared to integers; +by extension, integer enumerations of different types can also be compared +to each other:: + + >>> from enum import IntEnum + >>> class Shape(IntEnum): + ... circle = 1 + ... square = 2 + ... + >>> class Request(IntEnum): + ... post = 1 + ... get = 2 + ... + >>> Shape == 1 + False + >>> Shape.circle == 1 + True + >>> Shape.circle == Request.post + True + +However, they still can't be compared to standard ``Enum`` enumerations:: + + >>> class Shape(IntEnum): + ... circle = 1 + ... square = 2 + ... + >>> class Color(Enum): + ... red = 1 + ... green = 2 + ... + >>> Shape.circle == Color.red + False + +``IntEnum`` values behave like integers in other ways you'd expect:: + + >>> int(Shape.circle) + 1 + >>> ['a', 'b', 'c'][Shape.circle] + 'b' + >>> [i for i in range(Shape.square)] + [0, 1] + +For the vast majority of code, ``Enum`` is strongly recommended, +since ``IntEnum`` breaks some semantic promises of an enumeration (by +being comparable to integers, and thus by transitivity to other +unrelated enumerations). It should be used only in special cases where +there's no other choice; for example, when integer constants are +replaced with enumerations and backwards compatibility is required with code +that still expects integers. + + +Others +^^^^^^ + +While ``IntEnum`` is part of the ``enum`` module, it would be very +simple to implement independently:: + + class IntEnum(int, Enum): + pass + +This demonstrates how similar derived enumerations can be defined; for example +a ``StrEnum`` that mixes in ``str`` instead of ``int``. + +Some rules: + +1. When subclassing ``Enum``, mix-in types must appear before + ``Enum`` itself in the sequence of bases, as in the ``IntEnum`` + example above. +2. While ``Enum`` can have members of any type, once you mix in an + additional type, all the members must have values of that type, e.g. + ``int`` above. This restriction does not apply to mix-ins which only + add methods and don't specify another data type such as ``int`` or + ``str``. +3. When another data type is mixed in, the ``value`` attribute is *not the + same* as the enum member itself, although it is equivalant and will compare + equal. +4. %-style formatting: ``%s`` and ``%r`` call ``Enum``'s ``__str__`` and + ``__repr__`` respectively; other codes (such as ``%i`` or ``%h`` for + IntEnum) treat the enum member as its mixed-in type. + + Note: Prior to Python 3.4 there is a bug in ``str``'s %-formatting: ``int`` + subclasses are printed as strings and not numbers when the ``%d``, ``%i``, + or ``%u`` codes are used. +5. ``str.__format__`` (or ``format``) will use the mixed-in + type's ``__format__``. If the ``Enum``'s ``str`` or + ``repr`` is desired use the ``!s`` or ``!r`` ``str`` format codes. + + +Decorators +---------- + +unique +^^^^^^ + +A ``class`` decorator specifically for enumerations. It searches an +enumeration's ``__members__`` gathering any aliases it finds; if any are +found ``ValueError`` is raised with the details:: + + >>> @unique + ... class NoDupes(Enum): + ... first = 'one' + ... second = 'two' + ... third = 'two' + Traceback (most recent call last): + ... + ValueError: duplicate names found in <enum 'NoDupes'>: third -> second + + +Interesting examples +-------------------- + +While ``Enum`` and ``IntEnum`` are expected to cover the majority of +use-cases, they cannot cover them all. Here are recipes for some different +types of enumerations that can be used directly, or as examples for creating +one's own. + + +AutoNumber +^^^^^^^^^^ + +Avoids having to specify the value for each enumeration member:: + + >>> class AutoNumber(Enum): + ... def __new__(cls): + ... value = len(cls.__members__) + 1 + ... obj = object.__new__(cls) + ... obj._value_ = value + ... return obj + ... + >>> class Color(AutoNumber): + ... __order__ = "red green blue" # only needed in 2.x + ... red = () + ... green = () + ... blue = () + ... + >>> Color.green.value == 2 + True + +Note: + + The `__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's `__new__` which is used after + class creation for lookup of existing members. Due to the way Enums are + supposed to behave, there is no way to customize Enum's `__new__`. + + +UniqueEnum +^^^^^^^^^^ + +Raises an error if a duplicate member name is found instead of creating an +alias:: + + >>> class UniqueEnum(Enum): + ... def __init__(self, *args): + ... cls = self.__class__ + ... if any(self.value == e.value for e in cls): + ... a = self.name + ... e = cls(self.value).name + ... raise ValueError( + ... "aliases not allowed in UniqueEnum: %r --> %r" + ... % (a, e)) + ... + >>> class Color(UniqueEnum): + ... red = 1 + ... green = 2 + ... blue = 3 + ... grene = 2 + Traceback (most recent call last): + ... + ValueError: aliases not allowed in UniqueEnum: 'grene' --> 'green' + + +OrderedEnum +^^^^^^^^^^^ + +An ordered enumeration that is not based on ``IntEnum`` and so maintains +the normal ``Enum`` invariants (such as not being comparable to other +enumerations):: + + >>> class OrderedEnum(Enum): + ... def __ge__(self, other): + ... if self.__class__ is other.__class__: + ... return self._value_ >= other._value_ + ... return NotImplemented + ... def __gt__(self, other): + ... if self.__class__ is other.__class__: + ... return self._value_ > other._value_ + ... return NotImplemented + ... def __le__(self, other): + ... if self.__class__ is other.__class__: + ... return self._value_ <= other._value_ + ... return NotImplemented + ... def __lt__(self, other): + ... if self.__class__ is other.__class__: + ... return self._value_ < other._value_ + ... return NotImplemented + ... + >>> class Grade(OrderedEnum): + ... __ordered__ = 'A B C D F' + ... A = 5 + ... B = 4 + ... C = 3 + ... D = 2 + ... F = 1 + ... + >>> Grade.C < Grade.A + True + + +Planet +^^^^^^ + +If ``__new__`` or ``__init__`` is defined the value of the enum member +will be passed to those methods:: + + >>> class Planet(Enum): + ... MERCURY = (3.303e+23, 2.4397e6) + ... VENUS = (4.869e+24, 6.0518e6) + ... EARTH = (5.976e+24, 6.37814e6) + ... MARS = (6.421e+23, 3.3972e6) + ... JUPITER = (1.9e+27, 7.1492e7) + ... SATURN = (5.688e+26, 6.0268e7) + ... URANUS = (8.686e+25, 2.5559e7) + ... NEPTUNE = (1.024e+26, 2.4746e7) + ... def __init__(self, mass, radius): + ... self.mass = mass # in kilograms + ... self.radius = radius # in meters + ... @property + ... def surface_gravity(self): + ... # universal gravitational constant (m3 kg-1 s-2) + ... G = 6.67300E-11 + ... return G * self.mass / (self.radius * self.radius) + ... + >>> Planet.EARTH.value + (5.976e+24, 6378140.0) + >>> Planet.EARTH.surface_gravity + 9.802652743337129 + + +How are Enums different? +------------------------ + +Enums have a custom metaclass that affects many aspects of both derived Enum +classes and their instances (members). + + +Enum Classes +^^^^^^^^^^^^ + +The ``EnumMeta`` metaclass is responsible for providing the +``__contains__``, ``__dir__``, ``__iter__`` and other methods that +allow one to do things with an ``Enum`` class that fail on a typical +class, such as ``list(Color)`` or ``some_var in Color``. ``EnumMeta`` is +responsible for ensuring that various other methods on the final ``Enum`` +class are correct (such as ``__new__``, ``__getnewargs__``, +``__str__`` and ``__repr__``) + + +Enum Members (aka instances) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The most interesting thing about Enum members is that they are singletons. +``EnumMeta`` creates them all while it is creating the ``Enum`` +class itself, and then puts a custom ``__new__`` in place to ensure +that no new ones are ever instantiated by returning only the existing +member instances. + + +Finer Points +^^^^^^^^^^^^ + +Enum members are instances of an Enum class, and even though they are +accessible as ``EnumClass.member``, they are not accessible directly from +the member:: + + >>> Color.red + <Color.red: 1> + >>> Color.red.blue + Traceback (most recent call last): + ... + AttributeError: 'Color' object has no attribute 'blue' + +Likewise, ``__members__`` is only available on the class. + +In Python 3.x ``__members__`` is always an ``OrderedDict``, with the order being +the definition order. In Python 2.7 ``__members__`` is an ``OrderedDict`` if +``__order__`` was specified, and a plain ``dict`` otherwise. In all other Python +2.x versions ``__members__`` is a plain ``dict`` even if ``__order__`` was specified +as the ``OrderedDict`` type didn't exist yet. + +If you give your ``Enum`` subclass extra methods, like the `Planet`_ +class above, those methods will show up in a `dir` of the member, +but not of the class:: + + >>> dir(Planet) + ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', + 'VENUS', '__class__', '__doc__', '__members__', '__module__'] + >>> dir(Planet.EARTH) + ['__class__', '__doc__', '__module__', 'name', 'surface_gravity', 'value'] + +A ``__new__`` method will only be used for the creation of the +``Enum`` members -- after that it is replaced. This means if you wish to +change how ``Enum`` members are looked up you either have to write a +helper function or a ``classmethod``. diff --git a/scripts/external_libs/enum34-1.0.4/enum/enum.py b/scripts/external_libs/enum34-1.0.4/enum/enum.py new file mode 100644 index 00000000..6a327a8a --- /dev/null +++ b/scripts/external_libs/enum34-1.0.4/enum/enum.py @@ -0,0 +1,790 @@ +"""Python Enumerations""" + +import sys as _sys + +__all__ = ['Enum', 'IntEnum', 'unique'] + +version = 1, 0, 4 + +pyver = float('%s.%s' % _sys.version_info[:2]) + +try: + any +except NameError: + def any(iterable): + for element in iterable: + if element: + return True + return False + +try: + from collections import OrderedDict +except ImportError: + OrderedDict = None + +try: + basestring +except NameError: + # In Python 2 basestring is the ancestor of both str and unicode + # in Python 3 it's just str, but was missing in 3.1 + basestring = str + +try: + unicode +except NameError: + # In Python 3 unicode no longer exists (it's just str) + unicode = str + +class _RouteClassAttributeToGetattr(object): + """Route attribute access on a class to __getattr__. + + This is a descriptor, used to define attributes that act differently when + accessed through an instance and through a class. Instance access remains + normal, but access to an attribute through a class will be routed to the + class's __getattr__ method; this is done by raising AttributeError. + + """ + def __init__(self, fget=None): + self.fget = fget + + def __get__(self, instance, ownerclass=None): + if instance is None: + raise AttributeError() + return self.fget(instance) + + def __set__(self, instance, value): + raise AttributeError("can't set attribute") + + def __delete__(self, instance): + raise AttributeError("can't delete attribute") + + +def _is_descriptor(obj): + """Returns True if obj is a descriptor, False otherwise.""" + return ( + hasattr(obj, '__get__') or + hasattr(obj, '__set__') or + hasattr(obj, '__delete__')) + + +def _is_dunder(name): + """Returns True if a __dunder__ name, False otherwise.""" + return (name[:2] == name[-2:] == '__' and + name[2:3] != '_' and + name[-3:-2] != '_' and + len(name) > 4) + + +def _is_sunder(name): + """Returns True if a _sunder_ name, False otherwise.""" + return (name[0] == name[-1] == '_' and + name[1:2] != '_' and + name[-2:-1] != '_' and + len(name) > 2) + + +def _make_class_unpicklable(cls): + """Make the given class un-picklable.""" + def _break_on_call_reduce(self, protocol=None): + raise TypeError('%r cannot be pickled' % self) + cls.__reduce_ex__ = _break_on_call_reduce + cls.__module__ = '<unknown>' + + +class _EnumDict(dict): + """Track enum member order and ensure member names are not reused. + + EnumMeta will use the names found in self._member_names as the + enumeration member names. + + """ + def __init__(self): + super(_EnumDict, self).__init__() + self._member_names = [] + + def __setitem__(self, key, value): + """Changes anything not dundered or not a descriptor. + + If a descriptor is added with the same name as an enum member, the name + is removed from _member_names (this may leave a hole in the numerical + sequence of values). + + If an enum member name is used twice, an error is raised; duplicate + values are not checked for. + + Single underscore (sunder) names are reserved. + + Note: in 3.x __order__ is simply discarded as a not necessary piece + leftover from 2.x + + """ + if pyver >= 3.0 and key == '__order__': + return + if _is_sunder(key): + raise ValueError('_names_ are reserved for future Enum use') + elif _is_dunder(key): + pass + elif key in self._member_names: + # descriptor overwriting an enum? + raise TypeError('Attempted to reuse key: %r' % key) + elif not _is_descriptor(value): + if key in self: + # enum overwriting a descriptor? + raise TypeError('Key already defined as: %r' % self[key]) + self._member_names.append(key) + super(_EnumDict, self).__setitem__(key, value) + + +# Dummy value for Enum as EnumMeta explicity checks for it, but of course until +# EnumMeta finishes running the first time the Enum class doesn't exist. This +# is also why there are checks in EnumMeta like `if Enum is not None` +Enum = None + + +class EnumMeta(type): + """Metaclass for Enum""" + @classmethod + def __prepare__(metacls, cls, bases): + return _EnumDict() + + def __new__(metacls, cls, bases, classdict): + # an Enum class is final once enumeration items have been defined; it + # cannot be mixed with other types (int, float, etc.) if it has an + # inherited __new__ unless a new __new__ is defined (or the resulting + # class will fail). + if type(classdict) is dict: + original_dict = classdict + classdict = _EnumDict() + for k, v in original_dict.items(): + classdict[k] = v + + member_type, first_enum = metacls._get_mixins_(bases) + __new__, save_new, use_args = metacls._find_new_(classdict, member_type, + first_enum) + # save enum items into separate mapping so they don't get baked into + # the new class + members = dict((k, classdict[k]) for k in classdict._member_names) + for name in classdict._member_names: + del classdict[name] + + # py2 support for definition order + __order__ = classdict.get('__order__') + if __order__ is None: + if pyver < 3.0: + try: + __order__ = [name for (name, value) in sorted(members.items(), key=lambda item: item[1])] + except TypeError: + __order__ = [name for name in sorted(members.keys())] + else: + __order__ = classdict._member_names + else: + del classdict['__order__'] + if pyver < 3.0: + __order__ = __order__.replace(',', ' ').split() + aliases = [name for name in members if name not in __order__] + __order__ += aliases + + # check for illegal enum names (any others?) + invalid_names = set(members) & set(['mro']) + if invalid_names: + raise ValueError('Invalid enum member name(s): %s' % ( + ', '.join(invalid_names), )) + + # create our new Enum type + enum_class = super(EnumMeta, metacls).__new__(metacls, cls, bases, classdict) + enum_class._member_names_ = [] # names in random order + if OrderedDict is not None: + enum_class._member_map_ = OrderedDict() + else: + enum_class._member_map_ = {} # name->value map + enum_class._member_type_ = member_type + + # Reverse value->name map for hashable values. + enum_class._value2member_map_ = {} + + # instantiate them, checking for duplicates as we go + # we instantiate first instead of checking for duplicates first in case + # a custom __new__ is doing something funky with the values -- such as + # auto-numbering ;) + if __new__ is None: + __new__ = enum_class.__new__ + for member_name in __order__: + value = members[member_name] + if not isinstance(value, tuple): + args = (value, ) + else: + args = value + if member_type is tuple: # special case for tuple enums + args = (args, ) # wrap it one more time + if not use_args or not args: + enum_member = __new__(enum_class) + if not hasattr(enum_member, '_value_'): + enum_member._value_ = value + else: + enum_member = __new__(enum_class, *args) + if not hasattr(enum_member, '_value_'): + enum_member._value_ = member_type(*args) + value = enum_member._value_ + enum_member._name_ = member_name + enum_member.__objclass__ = enum_class + enum_member.__init__(*args) + # If another member with the same value was already defined, the + # new member becomes an alias to the existing one. + for name, canonical_member in enum_class._member_map_.items(): + if canonical_member.value == enum_member._value_: + enum_member = canonical_member + break + else: + # Aliases don't appear in member names (only in __members__). + enum_class._member_names_.append(member_name) + enum_class._member_map_[member_name] = enum_member + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_[value] = enum_member + except TypeError: + pass + + + # If a custom type is mixed into the Enum, and it does not know how + # to pickle itself, pickle.dumps will succeed but pickle.loads will + # fail. Rather than have the error show up later and possibly far + # from the source, sabotage the pickle protocol for this class so + # that pickle.dumps also fails. + # + # However, if the new class implements its own __reduce_ex__, do not + # sabotage -- it's on them to make sure it works correctly. We use + # __reduce_ex__ instead of any of the others as it is preferred by + # pickle over __reduce__, and it handles all pickle protocols. + unpicklable = False + if '__reduce_ex__' not in classdict: + if member_type is not object: + methods = ('__getnewargs_ex__', '__getnewargs__', + '__reduce_ex__', '__reduce__') + if not any(m in member_type.__dict__ for m in methods): + _make_class_unpicklable(enum_class) + unpicklable = True + + + # double check that repr and friends are not the mixin's or various + # things break (such as pickle) + for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'): + class_method = getattr(enum_class, name) + obj_method = getattr(member_type, name, None) + enum_method = getattr(first_enum, name, None) + if name not in classdict and class_method is not enum_method: + if name == '__reduce_ex__' and unpicklable: + continue + setattr(enum_class, name, enum_method) + + # method resolution and int's are not playing nice + # Python's less than 2.6 use __cmp__ + + if pyver < 2.6: + + if issubclass(enum_class, int): + setattr(enum_class, '__cmp__', getattr(int, '__cmp__')) + + elif pyver < 3.0: + + if issubclass(enum_class, int): + for method in ( + '__le__', + '__lt__', + '__gt__', + '__ge__', + '__eq__', + '__ne__', + '__hash__', + ): + setattr(enum_class, method, getattr(int, method)) + + # replace any other __new__ with our own (as long as Enum is not None, + # anyway) -- again, this is to support pickle + if Enum is not None: + # if the user defined their own __new__, save it before it gets + # clobbered in case they subclass later + if save_new: + setattr(enum_class, '__member_new__', enum_class.__dict__['__new__']) + setattr(enum_class, '__new__', Enum.__dict__['__new__']) + return enum_class + + def __call__(cls, value, names=None, module=None, type=None): + """Either returns an existing member, or creates a new enum class. + + This method is used both when an enum class is given a value to match + to an enumeration member (i.e. Color(3)) and for the functional API + (i.e. Color = Enum('Color', names='red green blue')). + + When used for the functional API: `module`, if set, will be stored in + the new class' __module__ attribute; `type`, if set, will be mixed in + as the first base class. + + Note: if `module` is not set this routine will attempt to discover the + calling module by walking the frame stack; if this is unsuccessful + the resulting class will not be pickleable. + + """ + if names is None: # simple value lookup + return cls.__new__(cls, value) + # otherwise, functional API: we're creating a new Enum type + return cls._create_(value, names, module=module, type=type) + + def __contains__(cls, member): + return isinstance(member, cls) and member.name in cls._member_map_ + + def __delattr__(cls, attr): + # nicer error message when someone tries to delete an attribute + # (see issue19025). + if attr in cls._member_map_: + raise AttributeError( + "%s: cannot delete Enum member." % cls.__name__) + super(EnumMeta, cls).__delattr__(attr) + + def __dir__(self): + return (['__class__', '__doc__', '__members__', '__module__'] + + self._member_names_) + + @property + def __members__(cls): + """Returns a mapping of member name->value. + + This mapping lists all enum members, including aliases. Note that this + is a copy of the internal mapping. + + """ + return cls._member_map_.copy() + + def __getattr__(cls, name): + """Return the enum member matching `name` + + We use __getattr__ instead of descriptors or inserting into the enum + class' __dict__ in order to support `name` and `value` being both + properties for enum members (which live in the class' __dict__) and + enum members themselves. + + """ + if _is_dunder(name): + raise AttributeError(name) + try: + return cls._member_map_[name] + except KeyError: + raise AttributeError(name) + + def __getitem__(cls, name): + return cls._member_map_[name] + + def __iter__(cls): + return (cls._member_map_[name] for name in cls._member_names_) + + def __reversed__(cls): + return (cls._member_map_[name] for name in reversed(cls._member_names_)) + + def __len__(cls): + return len(cls._member_names_) + + def __repr__(cls): + return "<enum %r>" % cls.__name__ + + def __setattr__(cls, name, value): + """Block attempts to reassign Enum members. + + A simple assignment to the class namespace only changes one of the + several possible ways to get an Enum member from the Enum class, + resulting in an inconsistent Enumeration. + + """ + member_map = cls.__dict__.get('_member_map_', {}) + if name in member_map: + raise AttributeError('Cannot reassign members.') + super(EnumMeta, cls).__setattr__(name, value) + + def _create_(cls, class_name, names=None, module=None, type=None): + """Convenience method to create a new Enum class. + + `names` can be: + + * A string containing member names, separated either with spaces or + commas. Values are auto-numbered from 1. + * An iterable of member names. Values are auto-numbered from 1. + * An iterable of (member name, value) pairs. + * A mapping of member name -> value. + + """ + if pyver < 3.0: + # if class_name is unicode, attempt a conversion to ASCII + if isinstance(class_name, unicode): + try: + class_name = class_name.encode('ascii') + except UnicodeEncodeError: + raise TypeError('%r is not representable in ASCII' % class_name) + metacls = cls.__class__ + if type is None: + bases = (cls, ) + else: + bases = (type, cls) + classdict = metacls.__prepare__(class_name, bases) + __order__ = [] + + # special processing needed for names? + if isinstance(names, basestring): + names = names.replace(',', ' ').split() + if isinstance(names, (tuple, list)) and isinstance(names[0], basestring): + names = [(e, i+1) for (i, e) in enumerate(names)] + + # Here, names is either an iterable of (name, value) or a mapping. + for item in names: + if isinstance(item, basestring): + member_name, member_value = item, names[item] + else: + member_name, member_value = item + classdict[member_name] = member_value + __order__.append(member_name) + # only set __order__ in classdict if name/value was not from a mapping + if not isinstance(item, basestring): + classdict['__order__'] = ' '.join(__order__) + enum_class = metacls.__new__(metacls, class_name, bases, classdict) + + # TODO: replace the frame hack if a blessed way to know the calling + # module is ever developed + if module is None: + try: + module = _sys._getframe(2).f_globals['__name__'] + except (AttributeError, ValueError): + pass + if module is None: + _make_class_unpicklable(enum_class) + else: + enum_class.__module__ = module + + return enum_class + + @staticmethod + def _get_mixins_(bases): + """Returns the type for creating enum members, and the first inherited + enum class. + + bases: the tuple of bases that was given to __new__ + + """ + if not bases or Enum is None: + return object, Enum + + + # double check that we are not subclassing a class with existing + # enumeration members; while we're at it, see if any other data + # type has been mixed in so we can use the correct __new__ + member_type = first_enum = None + for base in bases: + if (base is not Enum and + issubclass(base, Enum) and + base._member_names_): + raise TypeError("Cannot extend enumerations") + # base is now the last base in bases + if not issubclass(base, Enum): + raise TypeError("new enumerations must be created as " + "`ClassName([mixin_type,] enum_type)`") + + # get correct mix-in type (either mix-in type of Enum subclass, or + # first base if last base is Enum) + if not issubclass(bases[0], Enum): + member_type = bases[0] # first data type + first_enum = bases[-1] # enum type + else: + for base in bases[0].__mro__: + # most common: (IntEnum, int, Enum, object) + # possible: (<Enum 'AutoIntEnum'>, <Enum 'IntEnum'>, + # <class 'int'>, <Enum 'Enum'>, + # <class 'object'>) + if issubclass(base, Enum): + if first_enum is None: + first_enum = base + else: + if member_type is None: + member_type = base + + return member_type, first_enum + + if pyver < 3.0: + @staticmethod + def _find_new_(classdict, member_type, first_enum): + """Returns the __new__ to be used for creating the enum members. + + classdict: the class dictionary given to __new__ + member_type: the data type whose __new__ will be used by default + first_enum: enumeration to check for an overriding __new__ + + """ + # now find the correct __new__, checking to see of one was defined + # by the user; also check earlier enum classes in case a __new__ was + # saved as __member_new__ + __new__ = classdict.get('__new__', None) + if __new__: + return None, True, True # __new__, save_new, use_args + + N__new__ = getattr(None, '__new__') + O__new__ = getattr(object, '__new__') + if Enum is None: + E__new__ = N__new__ + else: + E__new__ = Enum.__dict__['__new__'] + # check all possibles for __member_new__ before falling back to + # __new__ + for method in ('__member_new__', '__new__'): + for possible in (member_type, first_enum): + try: + target = possible.__dict__[method] + except (AttributeError, KeyError): + target = getattr(possible, method, None) + if target not in [ + None, + N__new__, + O__new__, + E__new__, + ]: + if method == '__member_new__': + classdict['__new__'] = target + return None, False, True + if isinstance(target, staticmethod): + target = target.__get__(member_type) + __new__ = target + break + if __new__ is not None: + break + else: + __new__ = object.__new__ + + # if a non-object.__new__ is used then whatever value/tuple was + # assigned to the enum member name will be passed to __new__ and to the + # new enum member's __init__ + if __new__ is object.__new__: + use_args = False + else: + use_args = True + + return __new__, False, use_args + else: + @staticmethod + def _find_new_(classdict, member_type, first_enum): + """Returns the __new__ to be used for creating the enum members. + + classdict: the class dictionary given to __new__ + member_type: the data type whose __new__ will be used by default + first_enum: enumeration to check for an overriding __new__ + + """ + # now find the correct __new__, checking to see of one was defined + # by the user; also check earlier enum classes in case a __new__ was + # saved as __member_new__ + __new__ = classdict.get('__new__', None) + + # should __new__ be saved as __member_new__ later? + save_new = __new__ is not None + + if __new__ is None: + # check all possibles for __member_new__ before falling back to + # __new__ + for method in ('__member_new__', '__new__'): + for possible in (member_type, first_enum): + target = getattr(possible, method, None) + if target not in ( + None, + None.__new__, + object.__new__, + Enum.__new__, + ): + __new__ = target + break + if __new__ is not None: + break + else: + __new__ = object.__new__ + + # if a non-object.__new__ is used then whatever value/tuple was + # assigned to the enum member name will be passed to __new__ and to the + # new enum member's __init__ + if __new__ is object.__new__: + use_args = False + else: + use_args = True + + return __new__, save_new, use_args + + +######################################################## +# In order to support Python 2 and 3 with a single +# codebase we have to create the Enum methods separately +# and then use the `type(name, bases, dict)` method to +# create the class. +######################################################## +temp_enum_dict = {} +temp_enum_dict['__doc__'] = "Generic enumeration.\n\n Derive from this class to define new enumerations.\n\n" + +def __new__(cls, value): + # all enum instances are actually created during class construction + # without calling this method; this method is called by the metaclass' + # __call__ (i.e. Color(3) ), and by pickle + if type(value) is cls: + # For lookups like Color(Color.red) + value = value.value + #return value + # by-value search for a matching enum member + # see if it's in the reverse mapping (for hashable values) + try: + if value in cls._value2member_map_: + return cls._value2member_map_[value] + except TypeError: + # not there, now do long search -- O(n) behavior + for member in cls._member_map_.values(): + if member.value == value: + return member + raise ValueError("%s is not a valid %s" % (value, cls.__name__)) +temp_enum_dict['__new__'] = __new__ +del __new__ + +def __repr__(self): + return "<%s.%s: %r>" % ( + self.__class__.__name__, self._name_, self._value_) +temp_enum_dict['__repr__'] = __repr__ +del __repr__ + +def __str__(self): + return "%s.%s" % (self.__class__.__name__, self._name_) +temp_enum_dict['__str__'] = __str__ +del __str__ + +def __dir__(self): + added_behavior = [ + m + for cls in self.__class__.mro() + for m in cls.__dict__ + if m[0] != '_' + ] + return (['__class__', '__doc__', '__module__', ] + added_behavior) +temp_enum_dict['__dir__'] = __dir__ +del __dir__ + +def __format__(self, format_spec): + # mixed-in Enums should use the mixed-in type's __format__, otherwise + # we can get strange results with the Enum name showing up instead of + # the value + + # pure Enum branch + if self._member_type_ is object: + cls = str + val = str(self) + # mix-in branch + else: + cls = self._member_type_ + val = self.value + return cls.__format__(val, format_spec) +temp_enum_dict['__format__'] = __format__ +del __format__ + + +#################################### +# Python's less than 2.6 use __cmp__ + +if pyver < 2.6: + + def __cmp__(self, other): + if type(other) is self.__class__: + if self is other: + return 0 + return -1 + return NotImplemented + raise TypeError("unorderable types: %s() and %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__cmp__'] = __cmp__ + del __cmp__ + +else: + + def __le__(self, other): + raise TypeError("unorderable types: %s() <= %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__le__'] = __le__ + del __le__ + + def __lt__(self, other): + raise TypeError("unorderable types: %s() < %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__lt__'] = __lt__ + del __lt__ + + def __ge__(self, other): + raise TypeError("unorderable types: %s() >= %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__ge__'] = __ge__ + del __ge__ + + def __gt__(self, other): + raise TypeError("unorderable types: %s() > %s()" % (self.__class__.__name__, other.__class__.__name__)) + temp_enum_dict['__gt__'] = __gt__ + del __gt__ + + +def __eq__(self, other): + if type(other) is self.__class__: + return self is other + return NotImplemented +temp_enum_dict['__eq__'] = __eq__ +del __eq__ + +def __ne__(self, other): + if type(other) is self.__class__: + return self is not other + return NotImplemented +temp_enum_dict['__ne__'] = __ne__ +del __ne__ + +def __hash__(self): + return hash(self._name_) +temp_enum_dict['__hash__'] = __hash__ +del __hash__ + +def __reduce_ex__(self, proto): + return self.__class__, (self._value_, ) +temp_enum_dict['__reduce_ex__'] = __reduce_ex__ +del __reduce_ex__ + +# _RouteClassAttributeToGetattr is used to provide access to the `name` +# and `value` properties of enum members while keeping some measure of +# protection from modification, while still allowing for an enumeration +# to have members named `name` and `value`. This works because enumeration +# members are not set directly on the enum class -- __getattr__ is +# used to look them up. + +@_RouteClassAttributeToGetattr +def name(self): + return self._name_ +temp_enum_dict['name'] = name +del name + +@_RouteClassAttributeToGetattr +def value(self): + return self._value_ +temp_enum_dict['value'] = value +del value + +Enum = EnumMeta('Enum', (object, ), temp_enum_dict) +del temp_enum_dict + +# Enum has now been created +########################### + +class IntEnum(int, Enum): + """Enum where members are also (and must be) ints""" + + +def unique(enumeration): + """Class decorator that ensures only unique members exist in an enumeration.""" + duplicates = [] + for name, member in enumeration.__members__.items(): + if name != member.name: + duplicates.append((name, member.name)) + if duplicates: + duplicate_names = ', '.join( + ["%s -> %s" % (alias, name) for (alias, name) in duplicates] + ) + raise ValueError('duplicate names found in %r: %s' % + (enumeration, duplicate_names) + ) + return enumeration diff --git a/scripts/external_libs/enum34-1.0.4/enum/test_enum.py b/scripts/external_libs/enum34-1.0.4/enum/test_enum.py new file mode 100644 index 00000000..d7a97942 --- /dev/null +++ b/scripts/external_libs/enum34-1.0.4/enum/test_enum.py @@ -0,0 +1,1690 @@ +import enum +import sys +import unittest +from enum import Enum, IntEnum, unique, EnumMeta +from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL + +pyver = float('%s.%s' % sys.version_info[:2]) + +try: + any +except NameError: + def any(iterable): + for element in iterable: + if element: + return True + return False + +try: + unicode +except NameError: + unicode = str + +try: + from collections import OrderedDict +except ImportError: + OrderedDict = None + +# for pickle tests +try: + class Stooges(Enum): + LARRY = 1 + CURLY = 2 + MOE = 3 +except Exception: + Stooges = sys.exc_info()[1] + +try: + class IntStooges(int, Enum): + LARRY = 1 + CURLY = 2 + MOE = 3 +except Exception: + IntStooges = sys.exc_info()[1] + +try: + class FloatStooges(float, Enum): + LARRY = 1.39 + CURLY = 2.72 + MOE = 3.142596 +except Exception: + FloatStooges = sys.exc_info()[1] + +# for pickle test and subclass tests +try: + class StrEnum(str, Enum): + 'accepts only string values' + class Name(StrEnum): + BDFL = 'Guido van Rossum' + FLUFL = 'Barry Warsaw' +except Exception: + Name = sys.exc_info()[1] + +try: + Question = Enum('Question', 'who what when where why', module=__name__) +except Exception: + Question = sys.exc_info()[1] + +try: + Answer = Enum('Answer', 'him this then there because') +except Exception: + Answer = sys.exc_info()[1] + +try: + Theory = Enum('Theory', 'rule law supposition', qualname='spanish_inquisition') +except Exception: + Theory = sys.exc_info()[1] + +# for doctests +try: + class Fruit(Enum): + tomato = 1 + banana = 2 + cherry = 3 +except Exception: + pass + +def test_pickle_dump_load(assertion, source, target=None, + protocol=(0, HIGHEST_PROTOCOL)): + start, stop = protocol + failures = [] + for protocol in range(start, stop+1): + try: + if target is None: + assertion(loads(dumps(source, protocol=protocol)) is source) + else: + assertion(loads(dumps(source, protocol=protocol)), target) + except Exception: + exc, tb = sys.exc_info()[1:] + failures.append('%2d: %s' %(protocol, exc)) + if failures: + raise ValueError('Failed with protocols: %s' % ', '.join(failures)) + +def test_pickle_exception(assertion, exception, obj, + protocol=(0, HIGHEST_PROTOCOL)): + start, stop = protocol + failures = [] + for protocol in range(start, stop+1): + try: + assertion(exception, dumps, obj, protocol=protocol) + except Exception: + exc = sys.exc_info()[1] + failures.append('%d: %s %s' % (protocol, exc.__class__.__name__, exc)) + if failures: + raise ValueError('Failed with protocols: %s' % ', '.join(failures)) + + +class TestHelpers(unittest.TestCase): + # _is_descriptor, _is_sunder, _is_dunder + + def test_is_descriptor(self): + class foo: + pass + for attr in ('__get__','__set__','__delete__'): + obj = foo() + self.assertFalse(enum._is_descriptor(obj)) + setattr(obj, attr, 1) + self.assertTrue(enum._is_descriptor(obj)) + + def test_is_sunder(self): + for s in ('_a_', '_aa_'): + self.assertTrue(enum._is_sunder(s)) + + for s in ('a', 'a_', '_a', '__a', 'a__', '__a__', '_a__', '__a_', '_', + '__', '___', '____', '_____',): + self.assertFalse(enum._is_sunder(s)) + + def test_is_dunder(self): + for s in ('__a__', '__aa__'): + self.assertTrue(enum._is_dunder(s)) + for s in ('a', 'a_', '_a', '__a', 'a__', '_a_', '_a__', '__a_', '_', + '__', '___', '____', '_____',): + self.assertFalse(enum._is_dunder(s)) + + +class TestEnum(unittest.TestCase): + def setUp(self): + class Season(Enum): + SPRING = 1 + SUMMER = 2 + AUTUMN = 3 + WINTER = 4 + self.Season = Season + + class Konstants(float, Enum): + E = 2.7182818 + PI = 3.1415926 + TAU = 2 * PI + self.Konstants = Konstants + + class Grades(IntEnum): + A = 5 + B = 4 + C = 3 + D = 2 + F = 0 + self.Grades = Grades + + class Directional(str, Enum): + EAST = 'east' + WEST = 'west' + NORTH = 'north' + SOUTH = 'south' + self.Directional = Directional + + from datetime import date + class Holiday(date, Enum): + NEW_YEAR = 2013, 1, 1 + IDES_OF_MARCH = 2013, 3, 15 + self.Holiday = Holiday + + if pyver >= 2.6: # cannot specify custom `dir` on previous versions + def test_dir_on_class(self): + Season = self.Season + self.assertEqual( + set(dir(Season)), + set(['__class__', '__doc__', '__members__', '__module__', + 'SPRING', 'SUMMER', 'AUTUMN', 'WINTER']), + ) + + def test_dir_on_item(self): + Season = self.Season + self.assertEqual( + set(dir(Season.WINTER)), + set(['__class__', '__doc__', '__module__', 'name', 'value']), + ) + + def test_dir_on_sub_with_behavior_on_super(self): + # see issue22506 + class SuperEnum(Enum): + def invisible(self): + return "did you see me?" + class SubEnum(SuperEnum): + sample = 5 + self.assertEqual( + set(dir(SubEnum.sample)), + set(['__class__', '__doc__', '__module__', 'name', 'value', 'invisible']), + ) + + if pyver >= 2.7: # OrderedDict first available here + def test_members_is_ordereddict_if_ordered(self): + class Ordered(Enum): + __order__ = 'first second third' + first = 'bippity' + second = 'boppity' + third = 'boo' + self.assertTrue(type(Ordered.__members__) is OrderedDict) + + def test_members_is_ordereddict_if_not_ordered(self): + class Unordered(Enum): + this = 'that' + these = 'those' + self.assertTrue(type(Unordered.__members__) is OrderedDict) + + if pyver >= 3.0: # all objects are ordered in Python 2.x + def test_members_is_always_ordered(self): + class AlwaysOrdered(Enum): + first = 1 + second = 2 + third = 3 + self.assertTrue(type(AlwaysOrdered.__members__) is OrderedDict) + + def test_comparisons(self): + def bad_compare(): + Season.SPRING > 4 + Season = self.Season + self.assertNotEqual(Season.SPRING, 1) + self.assertRaises(TypeError, bad_compare) + + class Part(Enum): + SPRING = 1 + CLIP = 2 + BARREL = 3 + + self.assertNotEqual(Season.SPRING, Part.SPRING) + def bad_compare(): + Season.SPRING < Part.CLIP + self.assertRaises(TypeError, bad_compare) + + def test_enum_in_enum_out(self): + Season = self.Season + self.assertTrue(Season(Season.WINTER) is Season.WINTER) + + def test_enum_value(self): + Season = self.Season + self.assertEqual(Season.SPRING.value, 1) + + def test_intenum_value(self): + self.assertEqual(IntStooges.CURLY.value, 2) + + def test_enum(self): + Season = self.Season + lst = list(Season) + self.assertEqual(len(lst), len(Season)) + self.assertEqual(len(Season), 4, Season) + self.assertEqual( + [Season.SPRING, Season.SUMMER, Season.AUTUMN, Season.WINTER], lst) + + for i, season in enumerate('SPRING SUMMER AUTUMN WINTER'.split()): + i += 1 + e = Season(i) + self.assertEqual(e, getattr(Season, season)) + self.assertEqual(e.value, i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, season) + self.assertTrue(e in Season) + self.assertTrue(type(e) is Season) + self.assertTrue(isinstance(e, Season)) + self.assertEqual(str(e), 'Season.' + season) + self.assertEqual( + repr(e), + '<Season.%s: %s>' % (season, i), + ) + + def test_value_name(self): + Season = self.Season + self.assertEqual(Season.SPRING.name, 'SPRING') + self.assertEqual(Season.SPRING.value, 1) + def set_name(obj, new_value): + obj.name = new_value + def set_value(obj, new_value): + obj.value = new_value + self.assertRaises(AttributeError, set_name, Season.SPRING, 'invierno', ) + self.assertRaises(AttributeError, set_value, Season.SPRING, 2) + + def test_attribute_deletion(self): + class Season(Enum): + SPRING = 1 + SUMMER = 2 + AUTUMN = 3 + WINTER = 4 + + def spam(cls): + pass + + self.assertTrue(hasattr(Season, 'spam')) + del Season.spam + self.assertFalse(hasattr(Season, 'spam')) + + self.assertRaises(AttributeError, delattr, Season, 'SPRING') + self.assertRaises(AttributeError, delattr, Season, 'DRY') + self.assertRaises(AttributeError, delattr, Season.SPRING, 'name') + + def test_invalid_names(self): + def create_bad_class_1(): + class Wrong(Enum): + mro = 9 + def create_bad_class_2(): + class Wrong(Enum): + _reserved_ = 3 + self.assertRaises(ValueError, create_bad_class_1) + self.assertRaises(ValueError, create_bad_class_2) + + def test_contains(self): + Season = self.Season + self.assertTrue(Season.AUTUMN in Season) + self.assertTrue(3 not in Season) + + val = Season(3) + self.assertTrue(val in Season) + + class OtherEnum(Enum): + one = 1; two = 2 + self.assertTrue(OtherEnum.two not in Season) + + if pyver >= 2.6: # when `format` came into being + + def test_format_enum(self): + Season = self.Season + self.assertEqual('{0}'.format(Season.SPRING), + '{0}'.format(str(Season.SPRING))) + self.assertEqual( '{0:}'.format(Season.SPRING), + '{0:}'.format(str(Season.SPRING))) + self.assertEqual('{0:20}'.format(Season.SPRING), + '{0:20}'.format(str(Season.SPRING))) + self.assertEqual('{0:^20}'.format(Season.SPRING), + '{0:^20}'.format(str(Season.SPRING))) + self.assertEqual('{0:>20}'.format(Season.SPRING), + '{0:>20}'.format(str(Season.SPRING))) + self.assertEqual('{0:<20}'.format(Season.SPRING), + '{0:<20}'.format(str(Season.SPRING))) + + def test_format_enum_custom(self): + class TestFloat(float, Enum): + one = 1.0 + two = 2.0 + def __format__(self, spec): + return 'TestFloat success!' + self.assertEqual('{0}'.format(TestFloat.one), 'TestFloat success!') + + def assertFormatIsValue(self, spec, member): + self.assertEqual(spec.format(member), spec.format(member.value)) + + def test_format_enum_date(self): + Holiday = self.Holiday + self.assertFormatIsValue('{0}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{0:}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{0:20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{0:^20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{0:>20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{0:<20}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{0:%Y %m}', Holiday.IDES_OF_MARCH) + self.assertFormatIsValue('{0:%Y %m %M:00}', Holiday.IDES_OF_MARCH) + + def test_format_enum_float(self): + Konstants = self.Konstants + self.assertFormatIsValue('{0}', Konstants.TAU) + self.assertFormatIsValue('{0:}', Konstants.TAU) + self.assertFormatIsValue('{0:20}', Konstants.TAU) + self.assertFormatIsValue('{0:^20}', Konstants.TAU) + self.assertFormatIsValue('{0:>20}', Konstants.TAU) + self.assertFormatIsValue('{0:<20}', Konstants.TAU) + self.assertFormatIsValue('{0:n}', Konstants.TAU) + self.assertFormatIsValue('{0:5.2}', Konstants.TAU) + self.assertFormatIsValue('{0:f}', Konstants.TAU) + + def test_format_enum_int(self): + Grades = self.Grades + self.assertFormatIsValue('{0}', Grades.C) + self.assertFormatIsValue('{0:}', Grades.C) + self.assertFormatIsValue('{0:20}', Grades.C) + self.assertFormatIsValue('{0:^20}', Grades.C) + self.assertFormatIsValue('{0:>20}', Grades.C) + self.assertFormatIsValue('{0:<20}', Grades.C) + self.assertFormatIsValue('{0:+}', Grades.C) + self.assertFormatIsValue('{0:08X}', Grades.C) + self.assertFormatIsValue('{0:b}', Grades.C) + + def test_format_enum_str(self): + Directional = self.Directional + self.assertFormatIsValue('{0}', Directional.WEST) + self.assertFormatIsValue('{0:}', Directional.WEST) + self.assertFormatIsValue('{0:20}', Directional.WEST) + self.assertFormatIsValue('{0:^20}', Directional.WEST) + self.assertFormatIsValue('{0:>20}', Directional.WEST) + self.assertFormatIsValue('{0:<20}', Directional.WEST) + + def test_hash(self): + Season = self.Season + dates = {} + dates[Season.WINTER] = '1225' + dates[Season.SPRING] = '0315' + dates[Season.SUMMER] = '0704' + dates[Season.AUTUMN] = '1031' + self.assertEqual(dates[Season.AUTUMN], '1031') + + def test_enum_duplicates(self): + __order__ = "SPRING SUMMER AUTUMN WINTER" + class Season(Enum): + SPRING = 1 + SUMMER = 2 + AUTUMN = FALL = 3 + WINTER = 4 + ANOTHER_SPRING = 1 + lst = list(Season) + self.assertEqual( + lst, + [Season.SPRING, Season.SUMMER, + Season.AUTUMN, Season.WINTER, + ]) + self.assertTrue(Season.FALL is Season.AUTUMN) + self.assertEqual(Season.FALL.value, 3) + self.assertEqual(Season.AUTUMN.value, 3) + self.assertTrue(Season(3) is Season.AUTUMN) + self.assertTrue(Season(1) is Season.SPRING) + self.assertEqual(Season.FALL.name, 'AUTUMN') + self.assertEqual( + set([k for k,v in Season.__members__.items() if v.name != k]), + set(['FALL', 'ANOTHER_SPRING']), + ) + + if pyver >= 3.0: + cls = vars() + result = {'Enum':Enum} + exec("""def test_duplicate_name(self): + with self.assertRaises(TypeError): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + red = 4 + + with self.assertRaises(TypeError): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + def red(self): + return 'red' + + with self.assertRaises(TypeError): + class Color(Enum): + @property + + def red(self): + return 'redder' + red = 1 + green = 2 + blue = 3""", + result) + cls['test_duplicate_name'] = result['test_duplicate_name'] + + def test_enum_with_value_name(self): + class Huh(Enum): + name = 1 + value = 2 + self.assertEqual( + list(Huh), + [Huh.name, Huh.value], + ) + self.assertTrue(type(Huh.name) is Huh) + self.assertEqual(Huh.name.name, 'name') + self.assertEqual(Huh.name.value, 1) + + def test_intenum_from_scratch(self): + class phy(int, Enum): + pi = 3 + tau = 2 * pi + self.assertTrue(phy.pi < phy.tau) + + def test_intenum_inherited(self): + class IntEnum(int, Enum): + pass + class phy(IntEnum): + pi = 3 + tau = 2 * pi + self.assertTrue(phy.pi < phy.tau) + + def test_floatenum_from_scratch(self): + class phy(float, Enum): + pi = 3.1415926 + tau = 2 * pi + self.assertTrue(phy.pi < phy.tau) + + def test_floatenum_inherited(self): + class FloatEnum(float, Enum): + pass + class phy(FloatEnum): + pi = 3.1415926 + tau = 2 * pi + self.assertTrue(phy.pi < phy.tau) + + def test_strenum_from_scratch(self): + class phy(str, Enum): + pi = 'Pi' + tau = 'Tau' + self.assertTrue(phy.pi < phy.tau) + + def test_strenum_inherited(self): + class StrEnum(str, Enum): + pass + class phy(StrEnum): + pi = 'Pi' + tau = 'Tau' + self.assertTrue(phy.pi < phy.tau) + + def test_intenum(self): + class WeekDay(IntEnum): + SUNDAY = 1 + MONDAY = 2 + TUESDAY = 3 + WEDNESDAY = 4 + THURSDAY = 5 + FRIDAY = 6 + SATURDAY = 7 + + self.assertEqual(['a', 'b', 'c'][WeekDay.MONDAY], 'c') + self.assertEqual([i for i in range(WeekDay.TUESDAY)], [0, 1, 2]) + + lst = list(WeekDay) + self.assertEqual(len(lst), len(WeekDay)) + self.assertEqual(len(WeekDay), 7) + target = 'SUNDAY MONDAY TUESDAY WEDNESDAY THURSDAY FRIDAY SATURDAY' + target = target.split() + for i, weekday in enumerate(target): + i += 1 + e = WeekDay(i) + self.assertEqual(e, i) + self.assertEqual(int(e), i) + self.assertEqual(e.name, weekday) + self.assertTrue(e in WeekDay) + self.assertEqual(lst.index(e)+1, i) + self.assertTrue(0 < e < 8) + self.assertTrue(type(e) is WeekDay) + self.assertTrue(isinstance(e, int)) + self.assertTrue(isinstance(e, Enum)) + + def test_intenum_duplicates(self): + class WeekDay(IntEnum): + __order__ = 'SUNDAY MONDAY TUESDAY WEDNESDAY THURSDAY FRIDAY SATURDAY' + SUNDAY = 1 + MONDAY = 2 + TUESDAY = TEUSDAY = 3 + WEDNESDAY = 4 + THURSDAY = 5 + FRIDAY = 6 + SATURDAY = 7 + self.assertTrue(WeekDay.TEUSDAY is WeekDay.TUESDAY) + self.assertEqual(WeekDay(3).name, 'TUESDAY') + self.assertEqual([k for k,v in WeekDay.__members__.items() + if v.name != k], ['TEUSDAY', ]) + + def test_pickle_enum(self): + if isinstance(Stooges, Exception): + raise Stooges + test_pickle_dump_load(self.assertTrue, Stooges.CURLY) + test_pickle_dump_load(self.assertTrue, Stooges) + + def test_pickle_int(self): + if isinstance(IntStooges, Exception): + raise IntStooges + test_pickle_dump_load(self.assertTrue, IntStooges.CURLY) + test_pickle_dump_load(self.assertTrue, IntStooges) + + def test_pickle_float(self): + if isinstance(FloatStooges, Exception): + raise FloatStooges + test_pickle_dump_load(self.assertTrue, FloatStooges.CURLY) + test_pickle_dump_load(self.assertTrue, FloatStooges) + + def test_pickle_enum_function(self): + if isinstance(Answer, Exception): + raise Answer + test_pickle_dump_load(self.assertTrue, Answer.him) + test_pickle_dump_load(self.assertTrue, Answer) + + def test_pickle_enum_function_with_module(self): + if isinstance(Question, Exception): + raise Question + test_pickle_dump_load(self.assertTrue, Question.who) + test_pickle_dump_load(self.assertTrue, Question) + + if pyver >= 3.4: + def test_class_nested_enum_and_pickle_protocol_four(self): + # would normally just have this directly in the class namespace + class NestedEnum(Enum): + twigs = 'common' + shiny = 'rare' + + self.__class__.NestedEnum = NestedEnum + self.NestedEnum.__qualname__ = '%s.NestedEnum' % self.__class__.__name__ + test_pickle_exception( + self.assertRaises, PicklingError, self.NestedEnum.twigs, + protocol=(0, 3)) + test_pickle_dump_load(self.assertTrue, self.NestedEnum.twigs, + protocol=(4, HIGHEST_PROTOCOL)) + + def test_exploding_pickle(self): + BadPickle = Enum('BadPickle', 'dill sweet bread-n-butter') + enum._make_class_unpicklable(BadPickle) + globals()['BadPickle'] = BadPickle + test_pickle_exception(self.assertRaises, TypeError, BadPickle.dill) + test_pickle_exception(self.assertRaises, PicklingError, BadPickle) + + def test_string_enum(self): + class SkillLevel(str, Enum): + master = 'what is the sound of one hand clapping?' + journeyman = 'why did the chicken cross the road?' + apprentice = 'knock, knock!' + self.assertEqual(SkillLevel.apprentice, 'knock, knock!') + + def test_getattr_getitem(self): + class Period(Enum): + morning = 1 + noon = 2 + evening = 3 + night = 4 + self.assertTrue(Period(2) is Period.noon) + self.assertTrue(getattr(Period, 'night') is Period.night) + self.assertTrue(Period['morning'] is Period.morning) + + def test_getattr_dunder(self): + Season = self.Season + self.assertTrue(getattr(Season, '__hash__')) + + def test_iteration_order(self): + class Season(Enum): + __order__ = 'SUMMER WINTER AUTUMN SPRING' + SUMMER = 2 + WINTER = 4 + AUTUMN = 3 + SPRING = 1 + self.assertEqual( + list(Season), + [Season.SUMMER, Season.WINTER, Season.AUTUMN, Season.SPRING], + ) + + def test_iteration_order_with_unorderable_values(self): + class Complex(Enum): + a = complex(7, 9) + b = complex(3.14, 2) + c = complex(1, -1) + d = complex(-77, 32) + self.assertEqual( + list(Complex), + [Complex.a, Complex.b, Complex.c, Complex.d], + ) + + def test_programatic_function_string(self): + SummerMonth = Enum('SummerMonth', 'june july august') + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_string_list(self): + SummerMonth = Enum('SummerMonth', ['june', 'july', 'august']) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_iterable(self): + SummerMonth = Enum( + 'SummerMonth', + (('june', 1), ('july', 2), ('august', 3)) + ) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_from_dict(self): + SummerMonth = Enum( + 'SummerMonth', + dict((('june', 1), ('july', 2), ('august', 3))) + ) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + if pyver < 3.0: + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_type(self): + SummerMonth = Enum('SummerMonth', 'june july august', type=int) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_type_from_subclass(self): + SummerMonth = IntEnum('SummerMonth', 'june july august') + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate('june july august'.split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_unicode(self): + SummerMonth = Enum('SummerMonth', unicode('june july august')) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate(unicode('june july august').split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_unicode_list(self): + SummerMonth = Enum('SummerMonth', [unicode('june'), unicode('july'), unicode('august')]) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate(unicode('june july august').split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_unicode_iterable(self): + SummerMonth = Enum( + 'SummerMonth', + ((unicode('june'), 1), (unicode('july'), 2), (unicode('august'), 3)) + ) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate(unicode('june july august').split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_from_unicode_dict(self): + SummerMonth = Enum( + 'SummerMonth', + dict(((unicode('june'), 1), (unicode('july'), 2), (unicode('august'), 3))) + ) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + if pyver < 3.0: + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate(unicode('june july august').split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(int(e.value), i) + self.assertNotEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_unicode_type(self): + SummerMonth = Enum('SummerMonth', unicode('june july august'), type=int) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate(unicode('june july august').split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programatic_function_unicode_type_from_subclass(self): + SummerMonth = IntEnum('SummerMonth', unicode('june july august')) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate(unicode('june july august').split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(e, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_programmatic_function_unicode_class(self): + if pyver < 3.0: + class_names = unicode('SummerMonth'), 'S\xfcmm\xe9rM\xf6nth'.decode('latin1') + else: + class_names = 'SummerMonth', 'S\xfcmm\xe9rM\xf6nth' + for i, class_name in enumerate(class_names): + if pyver < 3.0 and i == 1: + self.assertRaises(TypeError, Enum, class_name, unicode('june july august')) + else: + SummerMonth = Enum(class_name, unicode('june july august')) + lst = list(SummerMonth) + self.assertEqual(len(lst), len(SummerMonth)) + self.assertEqual(len(SummerMonth), 3, SummerMonth) + self.assertEqual( + [SummerMonth.june, SummerMonth.july, SummerMonth.august], + lst, + ) + for i, month in enumerate(unicode('june july august').split()): + i += 1 + e = SummerMonth(i) + self.assertEqual(e.value, i) + self.assertEqual(e.name, month) + self.assertTrue(e in SummerMonth) + self.assertTrue(type(e) is SummerMonth) + + def test_subclassing(self): + if isinstance(Name, Exception): + raise Name + self.assertEqual(Name.BDFL, 'Guido van Rossum') + self.assertTrue(Name.BDFL, Name('Guido van Rossum')) + self.assertTrue(Name.BDFL is getattr(Name, 'BDFL')) + test_pickle_dump_load(self.assertTrue, Name.BDFL) + + def test_extending(self): + def bad_extension(): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + class MoreColor(Color): + cyan = 4 + magenta = 5 + yellow = 6 + self.assertRaises(TypeError, bad_extension) + + def test_exclude_methods(self): + class whatever(Enum): + this = 'that' + these = 'those' + def really(self): + return 'no, not %s' % self.value + self.assertFalse(type(whatever.really) is whatever) + self.assertEqual(whatever.this.really(), 'no, not that') + + def test_wrong_inheritance_order(self): + def wrong_inherit(): + class Wrong(Enum, str): + NotHere = 'error before this point' + self.assertRaises(TypeError, wrong_inherit) + + def test_intenum_transitivity(self): + class number(IntEnum): + one = 1 + two = 2 + three = 3 + class numero(IntEnum): + uno = 1 + dos = 2 + tres = 3 + self.assertEqual(number.one, numero.uno) + self.assertEqual(number.two, numero.dos) + self.assertEqual(number.three, numero.tres) + + def test_introspection(self): + class Number(IntEnum): + one = 100 + two = 200 + self.assertTrue(Number.one._member_type_ is int) + self.assertTrue(Number._member_type_ is int) + class String(str, Enum): + yarn = 'soft' + rope = 'rough' + wire = 'hard' + self.assertTrue(String.yarn._member_type_ is str) + self.assertTrue(String._member_type_ is str) + class Plain(Enum): + vanilla = 'white' + one = 1 + self.assertTrue(Plain.vanilla._member_type_ is object) + self.assertTrue(Plain._member_type_ is object) + + def test_wrong_enum_in_call(self): + class Monochrome(Enum): + black = 0 + white = 1 + class Gender(Enum): + male = 0 + female = 1 + self.assertRaises(ValueError, Monochrome, Gender.male) + + def test_wrong_enum_in_mixed_call(self): + class Monochrome(IntEnum): + black = 0 + white = 1 + class Gender(Enum): + male = 0 + female = 1 + self.assertRaises(ValueError, Monochrome, Gender.male) + + def test_mixed_enum_in_call_1(self): + class Monochrome(IntEnum): + black = 0 + white = 1 + class Gender(IntEnum): + male = 0 + female = 1 + self.assertTrue(Monochrome(Gender.female) is Monochrome.white) + + def test_mixed_enum_in_call_2(self): + class Monochrome(Enum): + black = 0 + white = 1 + class Gender(IntEnum): + male = 0 + female = 1 + self.assertTrue(Monochrome(Gender.male) is Monochrome.black) + + def test_flufl_enum(self): + class Fluflnum(Enum): + def __int__(self): + return int(self.value) + class MailManOptions(Fluflnum): + option1 = 1 + option2 = 2 + option3 = 3 + self.assertEqual(int(MailManOptions.option1), 1) + + def test_no_such_enum_member(self): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + self.assertRaises(ValueError, Color, 4) + self.assertRaises(KeyError, Color.__getitem__, 'chartreuse') + + def test_new_repr(self): + class Color(Enum): + red = 1 + green = 2 + blue = 3 + def __repr__(self): + return "don't you just love shades of %s?" % self.name + self.assertEqual( + repr(Color.blue), + "don't you just love shades of blue?", + ) + + def test_inherited_repr(self): + class MyEnum(Enum): + def __repr__(self): + return "My name is %s." % self.name + class MyIntEnum(int, MyEnum): + this = 1 + that = 2 + theother = 3 + self.assertEqual(repr(MyIntEnum.that), "My name is that.") + + def test_multiple_mixin_mro(self): + class auto_enum(EnumMeta): + def __new__(metacls, cls, bases, classdict): + original_dict = classdict + classdict = enum._EnumDict() + for k, v in original_dict.items(): + classdict[k] = v + temp = type(classdict)() + names = set(classdict._member_names) + i = 0 + for k in classdict._member_names: + v = classdict[k] + if v == (): + v = i + else: + i = v + i += 1 + temp[k] = v + for k, v in classdict.items(): + if k not in names: + temp[k] = v + return super(auto_enum, metacls).__new__( + metacls, cls, bases, temp) + + AutoNumberedEnum = auto_enum('AutoNumberedEnum', (Enum,), {}) + + AutoIntEnum = auto_enum('AutoIntEnum', (IntEnum,), {}) + + class TestAutoNumber(AutoNumberedEnum): + a = () + b = 3 + c = () + + class TestAutoInt(AutoIntEnum): + a = () + b = 3 + c = () + + def test_subclasses_with_getnewargs(self): + class NamedInt(int): + __qualname__ = 'NamedInt' # needed for pickle protocol 4 + def __new__(cls, *args): + _args = args + if len(args) < 1: + raise TypeError("name and value must be specified") + name, args = args[0], args[1:] + self = int.__new__(cls, *args) + self._intname = name + self._args = _args + return self + def __getnewargs__(self): + return self._args + @property + def __name__(self): + return self._intname + def __repr__(self): + # repr() is updated to include the name and type info + return "%s(%r, %s)" % (type(self).__name__, + self.__name__, + int.__repr__(self)) + def __str__(self): + # str() is unchanged, even if it relies on the repr() fallback + base = int + base_str = base.__str__ + if base_str.__objclass__ is object: + return base.__repr__(self) + return base_str(self) + # for simplicity, we only define one operator that + # propagates expressions + def __add__(self, other): + temp = int(self) + int( other) + if isinstance(self, NamedInt) and isinstance(other, NamedInt): + return NamedInt( + '(%s + %s)' % (self.__name__, other.__name__), + temp ) + else: + return temp + + class NEI(NamedInt, Enum): + __qualname__ = 'NEI' # needed for pickle protocol 4 + x = ('the-x', 1) + y = ('the-y', 2) + + self.assertTrue(NEI.__new__ is Enum.__new__) + self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)") + globals()['NamedInt'] = NamedInt + globals()['NEI'] = NEI + NI5 = NamedInt('test', 5) + self.assertEqual(NI5, 5) + test_pickle_dump_load(self.assertTrue, NI5, 5) + self.assertEqual(NEI.y.value, 2) + test_pickle_dump_load(self.assertTrue, NEI.y) + + if pyver >= 3.4: + def test_subclasses_with_getnewargs_ex(self): + class NamedInt(int): + __qualname__ = 'NamedInt' # needed for pickle protocol 4 + def __new__(cls, *args): + _args = args + if len(args) < 2: + raise TypeError("name and value must be specified") + name, args = args[0], args[1:] + self = int.__new__(cls, *args) + self._intname = name + self._args = _args + return self + def __getnewargs_ex__(self): + return self._args, {} + @property + def __name__(self): + return self._intname + def __repr__(self): + # repr() is updated to include the name and type info + return "{}({!r}, {})".format(type(self).__name__, + self.__name__, + int.__repr__(self)) + def __str__(self): + # str() is unchanged, even if it relies on the repr() fallback + base = int + base_str = base.__str__ + if base_str.__objclass__ is object: + return base.__repr__(self) + return base_str(self) + # for simplicity, we only define one operator that + # propagates expressions + def __add__(self, other): + temp = int(self) + int( other) + if isinstance(self, NamedInt) and isinstance(other, NamedInt): + return NamedInt( + '({0} + {1})'.format(self.__name__, other.__name__), + temp ) + else: + return temp + + class NEI(NamedInt, Enum): + __qualname__ = 'NEI' # needed for pickle protocol 4 + x = ('the-x', 1) + y = ('the-y', 2) + + + self.assertIs(NEI.__new__, Enum.__new__) + self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)") + globals()['NamedInt'] = NamedInt + globals()['NEI'] = NEI + NI5 = NamedInt('test', 5) + self.assertEqual(NI5, 5) + test_pickle_dump_load(self.assertEqual, NI5, 5, protocol=(4, HIGHEST_PROTOCOL)) + self.assertEqual(NEI.y.value, 2) + test_pickle_dump_load(self.assertTrue, NEI.y, protocol=(4, HIGHEST_PROTOCOL)) + + def test_subclasses_with_reduce(self): + class NamedInt(int): + __qualname__ = 'NamedInt' # needed for pickle protocol 4 + def __new__(cls, *args): + _args = args + if len(args) < 1: + raise TypeError("name and value must be specified") + name, args = args[0], args[1:] + self = int.__new__(cls, *args) + self._intname = name + self._args = _args + return self + def __reduce__(self): + return self.__class__, self._args + @property + def __name__(self): + return self._intname + def __repr__(self): + # repr() is updated to include the name and type info + return "%s(%r, %s)" % (type(self).__name__, + self.__name__, + int.__repr__(self)) + def __str__(self): + # str() is unchanged, even if it relies on the repr() fallback + base = int + base_str = base.__str__ + if base_str.__objclass__ is object: + return base.__repr__(self) + return base_str(self) + # for simplicity, we only define one operator that + # propagates expressions + def __add__(self, other): + temp = int(self) + int( other) + if isinstance(self, NamedInt) and isinstance(other, NamedInt): + return NamedInt( + '(%s + %s)' % (self.__name__, other.__name__), + temp ) + else: + return temp + + class NEI(NamedInt, Enum): + __qualname__ = 'NEI' # needed for pickle protocol 4 + x = ('the-x', 1) + y = ('the-y', 2) + + + self.assertTrue(NEI.__new__ is Enum.__new__) + self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)") + globals()['NamedInt'] = NamedInt + globals()['NEI'] = NEI + NI5 = NamedInt('test', 5) + self.assertEqual(NI5, 5) + test_pickle_dump_load(self.assertEqual, NI5, 5) + self.assertEqual(NEI.y.value, 2) + test_pickle_dump_load(self.assertTrue, NEI.y) + + def test_subclasses_with_reduce_ex(self): + class NamedInt(int): + __qualname__ = 'NamedInt' # needed for pickle protocol 4 + def __new__(cls, *args): + _args = args + if len(args) < 1: + raise TypeError("name and value must be specified") + name, args = args[0], args[1:] + self = int.__new__(cls, *args) + self._intname = name + self._args = _args + return self + def __reduce_ex__(self, proto): + return self.__class__, self._args + @property + def __name__(self): + return self._intname + def __repr__(self): + # repr() is updated to include the name and type info + return "%s(%r, %s)" % (type(self).__name__, + self.__name__, + int.__repr__(self)) + def __str__(self): + # str() is unchanged, even if it relies on the repr() fallback + base = int + base_str = base.__str__ + if base_str.__objclass__ is object: + return base.__repr__(self) + return base_str(self) + # for simplicity, we only define one operator that + # propagates expressions + def __add__(self, other): + temp = int(self) + int( other) + if isinstance(self, NamedInt) and isinstance(other, NamedInt): + return NamedInt( + '(%s + %s)' % (self.__name__, other.__name__), + temp ) + else: + return temp + + class NEI(NamedInt, Enum): + __qualname__ = 'NEI' # needed for pickle protocol 4 + x = ('the-x', 1) + y = ('the-y', 2) + + + self.assertTrue(NEI.__new__ is Enum.__new__) + self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)") + globals()['NamedInt'] = NamedInt + globals()['NEI'] = NEI + NI5 = NamedInt('test', 5) + self.assertEqual(NI5, 5) + test_pickle_dump_load(self.assertEqual, NI5, 5) + self.assertEqual(NEI.y.value, 2) + test_pickle_dump_load(self.assertTrue, NEI.y) + + def test_subclasses_without_direct_pickle_support(self): + class NamedInt(int): + __qualname__ = 'NamedInt' + def __new__(cls, *args): + _args = args + name, args = args[0], args[1:] + if len(args) == 0: + raise TypeError("name and value must be specified") + self = int.__new__(cls, *args) + self._intname = name + self._args = _args + return self + @property + def __name__(self): + return self._intname + def __repr__(self): + # repr() is updated to include the name and type info + return "%s(%r, %s)" % (type(self).__name__, + self.__name__, + int.__repr__(self)) + def __str__(self): + # str() is unchanged, even if it relies on the repr() fallback + base = int + base_str = base.__str__ + if base_str.__objclass__ is object: + return base.__repr__(self) + return base_str(self) + # for simplicity, we only define one operator that + # propagates expressions + def __add__(self, other): + temp = int(self) + int( other) + if isinstance(self, NamedInt) and isinstance(other, NamedInt): + return NamedInt( + '(%s + %s)' % (self.__name__, other.__name__), + temp ) + else: + return temp + + class NEI(NamedInt, Enum): + __qualname__ = 'NEI' + x = ('the-x', 1) + y = ('the-y', 2) + + self.assertTrue(NEI.__new__ is Enum.__new__) + self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)") + globals()['NamedInt'] = NamedInt + globals()['NEI'] = NEI + NI5 = NamedInt('test', 5) + self.assertEqual(NI5, 5) + self.assertEqual(NEI.y.value, 2) + test_pickle_exception(self.assertRaises, TypeError, NEI.x) + test_pickle_exception(self.assertRaises, PicklingError, NEI) + + def test_subclasses_without_direct_pickle_support_using_name(self): + class NamedInt(int): + __qualname__ = 'NamedInt' + def __new__(cls, *args): + _args = args + name, args = args[0], args[1:] + if len(args) == 0: + raise TypeError("name and value must be specified") + self = int.__new__(cls, *args) + self._intname = name + self._args = _args + return self + @property + def __name__(self): + return self._intname + def __repr__(self): + # repr() is updated to include the name and type info + return "%s(%r, %s)" % (type(self).__name__, + self.__name__, + int.__repr__(self)) + def __str__(self): + # str() is unchanged, even if it relies on the repr() fallback + base = int + base_str = base.__str__ + if base_str.__objclass__ is object: + return base.__repr__(self) + return base_str(self) + # for simplicity, we only define one operator that + # propagates expressions + def __add__(self, other): + temp = int(self) + int( other) + if isinstance(self, NamedInt) and isinstance(other, NamedInt): + return NamedInt( + '(%s + %s)' % (self.__name__, other.__name__), + temp ) + else: + return temp + + class NEI(NamedInt, Enum): + __qualname__ = 'NEI' + x = ('the-x', 1) + y = ('the-y', 2) + def __reduce_ex__(self, proto): + return getattr, (self.__class__, self._name_) + + self.assertTrue(NEI.__new__ is Enum.__new__) + self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)") + globals()['NamedInt'] = NamedInt + globals()['NEI'] = NEI + NI5 = NamedInt('test', 5) + self.assertEqual(NI5, 5) + self.assertEqual(NEI.y.value, 2) + test_pickle_dump_load(self.assertTrue, NEI.y) + test_pickle_dump_load(self.assertTrue, NEI) + + def test_tuple_subclass(self): + class SomeTuple(tuple, Enum): + __qualname__ = 'SomeTuple' + first = (1, 'for the money') + second = (2, 'for the show') + third = (3, 'for the music') + self.assertTrue(type(SomeTuple.first) is SomeTuple) + self.assertTrue(isinstance(SomeTuple.second, tuple)) + self.assertEqual(SomeTuple.third, (3, 'for the music')) + globals()['SomeTuple'] = SomeTuple + test_pickle_dump_load(self.assertTrue, SomeTuple.first) + + def test_duplicate_values_give_unique_enum_items(self): + class AutoNumber(Enum): + __order__ = 'enum_m enum_d enum_y' + enum_m = () + enum_d = () + enum_y = () + def __new__(cls): + value = len(cls.__members__) + 1 + obj = object.__new__(cls) + obj._value_ = value + return obj + def __int__(self): + return int(self._value_) + self.assertEqual(int(AutoNumber.enum_d), 2) + self.assertEqual(AutoNumber.enum_y.value, 3) + self.assertTrue(AutoNumber(1) is AutoNumber.enum_m) + self.assertEqual( + list(AutoNumber), + [AutoNumber.enum_m, AutoNumber.enum_d, AutoNumber.enum_y], + ) + + def test_inherited_new_from_enhanced_enum(self): + class AutoNumber2(Enum): + def __new__(cls): + value = len(cls.__members__) + 1 + obj = object.__new__(cls) + obj._value_ = value + return obj + def __int__(self): + return int(self._value_) + class Color(AutoNumber2): + __order__ = 'red green blue' + red = () + green = () + blue = () + self.assertEqual(len(Color), 3, "wrong number of elements: %d (should be %d)" % (len(Color), 3)) + self.assertEqual(list(Color), [Color.red, Color.green, Color.blue]) + if pyver >= 3.0: + self.assertEqual(list(map(int, Color)), [1, 2, 3]) + + def test_inherited_new_from_mixed_enum(self): + class AutoNumber3(IntEnum): + def __new__(cls): + value = len(cls.__members__) + 1 + obj = int.__new__(cls, value) + obj._value_ = value + return obj + class Color(AutoNumber3): + red = () + green = () + blue = () + self.assertEqual(len(Color), 3, "wrong number of elements: %d (should be %d)" % (len(Color), 3)) + Color.red + Color.green + Color.blue + + def test_ordered_mixin(self): + class OrderedEnum(Enum): + def __ge__(self, other): + if self.__class__ is other.__class__: + return self._value_ >= other._value_ + return NotImplemented + def __gt__(self, other): + if self.__class__ is other.__class__: + return self._value_ > other._value_ + return NotImplemented + def __le__(self, other): + if self.__class__ is other.__class__: + return self._value_ <= other._value_ + return NotImplemented + def __lt__(self, other): + if self.__class__ is other.__class__: + return self._value_ < other._value_ + return NotImplemented + class Grade(OrderedEnum): + __order__ = 'A B C D F' + A = 5 + B = 4 + C = 3 + D = 2 + F = 1 + self.assertEqual(list(Grade), [Grade.A, Grade.B, Grade.C, Grade.D, Grade.F]) + self.assertTrue(Grade.A > Grade.B) + self.assertTrue(Grade.F <= Grade.C) + self.assertTrue(Grade.D < Grade.A) + self.assertTrue(Grade.B >= Grade.B) + + def test_extending2(self): + def bad_extension(): + class Shade(Enum): + def shade(self): + print(self.name) + class Color(Shade): + red = 1 + green = 2 + blue = 3 + class MoreColor(Color): + cyan = 4 + magenta = 5 + yellow = 6 + self.assertRaises(TypeError, bad_extension) + + def test_extending3(self): + class Shade(Enum): + def shade(self): + return self.name + class Color(Shade): + def hex(self): + return '%s hexlified!' % self.value + class MoreColor(Color): + cyan = 4 + magenta = 5 + yellow = 6 + self.assertEqual(MoreColor.magenta.hex(), '5 hexlified!') + + def test_no_duplicates(self): + def bad_duplicates(): + class UniqueEnum(Enum): + def __init__(self, *args): + cls = self.__class__ + if any(self.value == e.value for e in cls): + a = self.name + e = cls(self.value).name + raise ValueError( + "aliases not allowed in UniqueEnum: %r --> %r" + % (a, e) + ) + class Color(UniqueEnum): + red = 1 + green = 2 + blue = 3 + class Color(UniqueEnum): + red = 1 + green = 2 + blue = 3 + grene = 2 + self.assertRaises(ValueError, bad_duplicates) + + def test_reversed(self): + self.assertEqual( + list(reversed(self.Season)), + [self.Season.WINTER, self.Season.AUTUMN, self.Season.SUMMER, + self.Season.SPRING] + ) + + def test_init(self): + class Planet(Enum): + MERCURY = (3.303e+23, 2.4397e6) + VENUS = (4.869e+24, 6.0518e6) + EARTH = (5.976e+24, 6.37814e6) + MARS = (6.421e+23, 3.3972e6) + JUPITER = (1.9e+27, 7.1492e7) + SATURN = (5.688e+26, 6.0268e7) + URANUS = (8.686e+25, 2.5559e7) + NEPTUNE = (1.024e+26, 2.4746e7) + def __init__(self, mass, radius): + self.mass = mass # in kilograms + self.radius = radius # in meters + @property + def surface_gravity(self): + # universal gravitational constant (m3 kg-1 s-2) + G = 6.67300E-11 + return G * self.mass / (self.radius * self.radius) + self.assertEqual(round(Planet.EARTH.surface_gravity, 2), 9.80) + self.assertEqual(Planet.EARTH.value, (5.976e+24, 6.37814e6)) + + def test_nonhash_value(self): + class AutoNumberInAList(Enum): + def __new__(cls): + value = [len(cls.__members__) + 1] + obj = object.__new__(cls) + obj._value_ = value + return obj + class ColorInAList(AutoNumberInAList): + __order__ = 'red green blue' + red = () + green = () + blue = () + self.assertEqual(list(ColorInAList), [ColorInAList.red, ColorInAList.green, ColorInAList.blue]) + self.assertEqual(ColorInAList.red.value, [1]) + self.assertEqual(ColorInAList([1]), ColorInAList.red) + + def test_conflicting_types_resolved_in_new(self): + class LabelledIntEnum(int, Enum): + def __new__(cls, *args): + value, label = args + obj = int.__new__(cls, value) + obj.label = label + obj._value_ = value + return obj + + class LabelledList(LabelledIntEnum): + unprocessed = (1, "Unprocessed") + payment_complete = (2, "Payment Complete") + + self.assertEqual(list(LabelledList), [LabelledList.unprocessed, LabelledList.payment_complete]) + self.assertEqual(LabelledList.unprocessed, 1) + self.assertEqual(LabelledList(1), LabelledList.unprocessed) + +class TestUnique(unittest.TestCase): + """2.4 doesn't allow class decorators, use function syntax.""" + + def test_unique_clean(self): + class Clean(Enum): + one = 1 + two = 'dos' + tres = 4.0 + unique(Clean) + class Cleaner(IntEnum): + single = 1 + double = 2 + triple = 3 + unique(Cleaner) + + def test_unique_dirty(self): + try: + class Dirty(Enum): + __order__ = 'one two tres' + one = 1 + two = 'dos' + tres = 1 + unique(Dirty) + except ValueError: + exc = sys.exc_info()[1] + message = exc.args[0] + self.assertTrue('tres -> one' in message) + + try: + class Dirtier(IntEnum): + __order__ = 'single double triple turkey' + single = 1 + double = 1 + triple = 3 + turkey = 3 + unique(Dirtier) + except ValueError: + exc = sys.exc_info()[1] + message = exc.args[0] + self.assertTrue('double -> single' in message) + self.assertTrue('turkey -> triple' in message) + + +class TestMe(unittest.TestCase): + + pass + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/external_libs/enum34-1.0.4/setup.py b/scripts/external_libs/enum34-1.0.4/setup.py new file mode 100644 index 00000000..4cb9c691 --- /dev/null +++ b/scripts/external_libs/enum34-1.0.4/setup.py @@ -0,0 +1,44 @@ +import os +import sys +from distutils.core import setup + +if sys.version_info[:2] < (2, 7): + required = ['ordereddict'] +else: + required = [] + +long_desc = open('enum/doc/enum.rst').read() + +setup( name='enum34', + version='1.0.4', + url='https://pypi.python.org/pypi/enum34', + packages=['enum'], + package_data={ + 'enum' : [ + 'LICENSE', + 'README', + 'doc/enum.rst', + 'doc/enum.pdf', + 'test_enum.py', + ] + }, + license='BSD License', + description='Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4', + long_description=long_desc, + provides=['enum'], + install_requires=required, + author='Ethan Furman', + author_email='ethan@stoneleaf.us', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Programming Language :: Python', + 'Topic :: Software Development', + 'Programming Language :: Python :: 2.4', + 'Programming Language :: Python :: 2.5', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + ], + ) diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/LICENSE.txt b/scripts/external_libs/jsonrpclib-pelix-0.2.5/LICENSE.txt new file mode 100644 index 00000000..51fca54c --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/LICENSE.txt @@ -0,0 +1,11 @@ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/MANIFEST.in b/scripts/external_libs/jsonrpclib-pelix-0.2.5/MANIFEST.in new file mode 100644 index 00000000..eb0014ad --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/MANIFEST.in @@ -0,0 +1,2 @@ +include *.txt +include README.rst diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/PKG-INFO b/scripts/external_libs/jsonrpclib-pelix-0.2.5/PKG-INFO new file mode 100644 index 00000000..5dce6b1c --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/PKG-INFO @@ -0,0 +1,460 @@ +Metadata-Version: 1.1 +Name: jsonrpclib-pelix +Version: 0.2.5 +Summary: This project is an implementation of the JSON-RPC v2.0 specification (backwards-compatible) as a client library, for Python 2.6+ and Python 3.This version is a fork of jsonrpclib by Josh Marshall, usable with Pelix remote services. +Home-page: http://github.com/tcalmant/jsonrpclib/ +Author: Thomas Calmant +Author-email: thomas.calmant+github@gmail.com +License: Apache License 2.0 +Description: JSONRPClib (patched for Pelix) + ############################## + + .. image:: https://pypip.in/license/jsonrpclib-pelix/badge.svg + :target: https://pypi.python.org/pypi/jsonrpclib-pelix/ + + .. image:: https://travis-ci.org/tcalmant/jsonrpclib.svg?branch=master + :target: https://travis-ci.org/tcalmant/jsonrpclib + + .. image:: https://coveralls.io/repos/tcalmant/jsonrpclib/badge.svg?branch=master + :target: https://coveralls.io/r/tcalmant/jsonrpclib?branch=master + + + This library is an implementation of the JSON-RPC specification. + It supports both the original 1.0 specification, as well as the + new (proposed) 2.0 specification, which includes batch submission, keyword + arguments, etc. + + It is licensed under the Apache License, Version 2.0 + (http://www.apache.org/licenses/LICENSE-2.0.html). + + + About this version + ****************** + + This is a patched version of the original ``jsonrpclib`` project by + Josh Marshall, available at https://github.com/joshmarshall/jsonrpclib. + + The suffix *-pelix* only indicates that this version works with Pelix Remote + Services, but it is **not** a Pelix specific implementation. + + * This version adds support for Python 3, staying compatible with Python 2. + * It is now possible to use the dispatch_method argument while extending + the SimpleJSONRPCDispatcher, to use a custom dispatcher. + This allows to use this package by Pelix Remote Services. + * It can use thread pools to control the number of threads spawned to handle + notification requests and clients connections. + * The modifications added in other forks of this project have been added: + + * From https://github.com/drdaeman/jsonrpclib: + + * Improved JSON-RPC 1.0 support + * Less strict error response handling + + * From https://github.com/tuomassalo/jsonrpclib: + + * In case of a non-pre-defined error, raise an AppError and give access to + *error.data* + + * From https://github.com/dejw/jsonrpclib: + + * Custom headers can be sent with request and associated tests + + * The support for Unix sockets has been removed, as it is not trivial to convert + to Python 3 (and I don't use them) + * This version cannot be installed with the original ``jsonrpclib``, as it uses + the same package name. + + + Summary + ******* + + This library implements the JSON-RPC 2.0 proposed specification in pure Python. + It is designed to be as compatible with the syntax of ``xmlrpclib`` as possible + (it extends where possible), so that projects using ``xmlrpclib`` could easily + be modified to use JSON and experiment with the differences. + + It is backwards-compatible with the 1.0 specification, and supports all of the + new proposed features of 2.0, including: + + * Batch submission (via MultiCall) + * Keyword arguments + * Notifications (both in a batch and 'normal') + * Class translation using the ``__jsonclass__`` key. + + I've added a "SimpleJSONRPCServer", which is intended to emulate the + "SimpleXMLRPCServer" from the default Python distribution. + + + Requirements + ************ + + It supports ``cjson`` and ``simplejson``, and looks for the parsers in that + order (searching first for ``cjson``, then for the *built-in* ``json`` in 2.6+, + and then the ``simplejson`` external library). + One of these must be installed to use this library, although if you have a + standard distribution of 2.6+, you should already have one. + Keep in mind that ``cjson`` is supposed to be the quickest, I believe, so if + you are going for full-on optimization you may want to pick it up. + + Since library uses ``contextlib`` module, you should have at least Python 2.5 + installed. + + + Installation + ************ + + You can install this from PyPI with one of the following commands (sudo + may be required): + + .. code-block:: console + + easy_install jsonrpclib-pelix + pip install jsonrpclib-pelix + + Alternatively, you can download the source from the GitHub repository + at http://github.com/tcalmant/jsonrpclib and manually install it + with the following commands: + + .. code-block:: console + + git clone git://github.com/tcalmant/jsonrpclib.git + cd jsonrpclib + python setup.py install + + + SimpleJSONRPCServer + ******************* + + This is identical in usage (or should be) to the SimpleXMLRPCServer in the + Python standard library. Some of the differences in features are that it + obviously supports notification, batch calls, class translation (if left on), + etc. + Note: The import line is slightly different from the regular SimpleXMLRPCServer, + since the SimpleJSONRPCServer is distributed within the ``jsonrpclib`` library. + + .. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + + server = SimpleJSONRPCServer(('localhost', 8080)) + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_function(lambda x: x, 'ping') + server.serve_forever() + + To start protect the server with SSL, use the following snippet: + + .. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + + # Setup the SSL socket + server = SimpleJSONRPCServer(('localhost', 8080), bind_and_activate=False) + server.socket = ssl.wrap_socket(server.socket, certfile='server.pem', + server_side=True) + server.server_bind() + server.server_activate() + + # ... register functions + # Start the server + server.serve_forever() + + + Notification Thread Pool + ======================== + + By default, notification calls are handled in the request handling thread. + It is possible to use a thread pool to handle them, by giving it to the server + using the ``set_notification_pool()`` method: + + .. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + from jsonrpclib.threadpool import ThreadPool + + # Setup the thread pool: between 0 and 10 threads + pool = ThreadPool(max_threads=10, min_threads=0) + + # Don't forget to start it + pool.start() + + # Setup the server + server = SimpleJSONRPCServer(('localhost', 8080), config) + server.set_notification_pool(pool) + + # Register methods + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_function(lambda x: x, 'ping') + + try: + server.serve_forever() + finally: + # Stop the thread pool (let threads finish their current task) + pool.stop() + server.set_notification_pool(None) + + + Threaded server + =============== + + It is also possible to use a thread pool to handle clients requests, using the + ``PooledJSONRPCServer`` class. + By default, this class uses pool of 0 to 30 threads. A custom pool can be given + with the ``thread_pool`` parameter of the class constructor. + + The notification pool and the request pool are different: by default, a server + with a request pool doesn't have a notification pool. + + .. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import PooledJSONRPCServer + from jsonrpclib.threadpool import ThreadPool + + # Setup the notification and request pools + nofif_pool = ThreadPool(max_threads=10, min_threads=0) + request_pool = ThreadPool(max_threads=50, min_threads=10) + + # Don't forget to start them + nofif_pool.start() + request_pool.start() + + # Setup the server + server = PooledJSONRPCServer(('localhost', 8080), config, + thread_pool=request_pool) + server.set_notification_pool(nofif_pool) + + # Register methods + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_function(lambda x: x, 'ping') + + try: + server.serve_forever() + finally: + # Stop the thread pools (let threads finish their current task) + request_pool.stop() + nofif_pool.stop() + server.set_notification_pool(None) + + Client Usage + ************ + + This is (obviously) taken from a console session. + + .. code-block:: python + + >>> import jsonrpclib + >>> server = jsonrpclib.ServerProxy('http://localhost:8080') + >>> server.add(5,6) + 11 + >>> server.add(x=5, y=10) + 15 + >>> server._notify.add(5,6) + # No result returned... + >>> batch = jsonrpclib.MultiCall(server) + >>> batch.add(5, 6) + >>> batch.ping({'key':'value'}) + >>> batch._notify.add(4, 30) + >>> results = batch() + >>> for result in results: + >>> ... print(result) + 11 + {'key': 'value'} + # Note that there are only two responses -- this is according to spec. + + # Clean up + >>> server('close')() + + # Using client history + >>> history = jsonrpclib.history.History() + >>> server = jsonrpclib.ServerProxy('http://localhost:8080', history=history) + >>> server.add(5,6) + 11 + >>> print(history.request) + {"id": "f682b956-c8e1-4506-9db4-29fe8bc9fcaa", "jsonrpc": "2.0", + "method": "add", "params": [5, 6]} + >>> print(history.response) + {"id": "f682b956-c8e1-4506-9db4-29fe8bc9fcaa", "jsonrpc": "2.0", + "result": 11} + + # Clean up + >>> server('close')() + + If you need 1.0 functionality, there are a bunch of places you can pass that in, + although the best is just to give a specific configuration to + ``jsonrpclib.ServerProxy``: + + .. code-block:: python + + >>> import jsonrpclib + >>> jsonrpclib.config.DEFAULT.version + 2.0 + >>> config = jsonrpclib.config.Config(version=1.0) + >>> history = jsonrpclib.history.History() + >>> server = jsonrpclib.ServerProxy('http://localhost:8080', config=config, + history=history) + >>> server.add(7, 10) + 17 + >>> print(history.request) + {"id": "827b2923-5b37-49a5-8b36-e73920a16d32", + "method": "add", "params": [7, 10]} + >>> print(history.response) + {"id": "827b2923-5b37-49a5-8b36-e73920a16d32", "error": null, "result": 17} + >>> server('close')() + + The equivalent ``loads`` and ``dumps`` functions also exist, although with minor + modifications. The ``dumps`` arguments are almost identical, but it adds three + arguments: ``rpcid`` for the 'id' key, ``version`` to specify the JSON-RPC + compatibility, and ``notify`` if it's a request that you want to be a + notification. + + Additionally, the ``loads`` method does not return the params and method like + ``xmlrpclib``, but instead a.) parses for errors, raising ProtocolErrors, and + b.) returns the entire structure of the request / response for manual parsing. + + + Additional headers + ****************** + + If your remote service requires custom headers in request, you can pass them + as as a ``headers`` keyword argument, when creating the ``ServerProxy``: + + .. code-block:: python + + >>> import jsonrpclib + >>> server = jsonrpclib.ServerProxy("http://localhost:8080", + headers={'X-Test' : 'Test'}) + + You can also put additional request headers only for certain method invocation: + + .. code-block:: python + + >>> import jsonrpclib + >>> server = jsonrpclib.Server("http://localhost:8080") + >>> with server._additional_headers({'X-Test' : 'Test'}) as test_server: + ... test_server.ping(42) + ... + >>> # X-Test header will be no longer sent in requests + + Of course ``_additional_headers`` contexts can be nested as well. + + + Class Translation + ***************** + + I've recently added "automatic" class translation support, although it is + turned off by default. This can be devastatingly slow if improperly used, so + the following is just a short list of things to keep in mind when using it. + + * Keep It (the object) Simple Stupid. (for exceptions, keep reading.) + * Do not require init params (for exceptions, keep reading) + * Getter properties without setters could be dangerous (read: not tested) + + If any of the above are issues, use the _serialize method. (see usage below) + The server and client must BOTH have use_jsonclass configuration item on and + they must both have access to the same libraries used by the objects for + this to work. + + If you have excessively nested arguments, it would be better to turn off the + translation and manually invoke it on specific objects using + ``jsonrpclib.jsonclass.dump`` / ``jsonrpclib.jsonclass.load`` (since the default + behavior recursively goes through attributes and lists / dicts / tuples). + + Sample file: *test_obj.py* + + .. code-block:: python + + # This object is /very/ simple, and the system will look through the + # attributes and serialize what it can. + class TestObj(object): + foo = 'bar' + + # This object requires __init__ params, so it uses the _serialize method + # and returns a tuple of init params and attribute values (the init params + # can be a dict or a list, but the attribute values must be a dict.) + class TestSerial(object): + foo = 'bar' + def __init__(self, *args): + self.args = args + def _serialize(self): + return (self.args, {'foo':self.foo,}) + + * Sample usage + + .. code-block:: python + + >>> import jsonrpclib + >>> import test_obj + + # History is used only to print the serialized form of beans + >>> history = jsonrpclib.history.History() + >>> testobj1 = test_obj.TestObj() + >>> testobj2 = test_obj.TestSerial() + >>> server = jsonrpclib.Server('http://localhost:8080', history=history) + + # The 'ping' just returns whatever is sent + >>> ping1 = server.ping(testobj1) + >>> ping2 = server.ping(testobj2) + + >>> print(history.request) + {"id": "7805f1f9-9abd-49c6-81dc-dbd47229fe13", "jsonrpc": "2.0", + "method": "ping", "params": [{"__jsonclass__": + ["test_obj.TestSerial", []], "foo": "bar"} + ]} + >>> print(history.response) + {"id": "7805f1f9-9abd-49c6-81dc-dbd47229fe13", "jsonrpc": "2.0", + "result": {"__jsonclass__": ["test_obj.TestSerial", []], "foo": "bar"}} + + This behavior is turned by default. To deactivate it, just set the + ``use_jsonclass`` member of a server ``Config`` to False. + If you want to use a per-class serialization method, set its name in the + ``serialize_method`` member of a server ``Config``. + Finally, if you are using classes that you have defined in the implementation + (as in, not a separate library), you'll need to add those (on BOTH the server + and the client) using the ``config.classes.add()`` method. + + Feedback on this "feature" is very, VERY much appreciated. + + Why JSON-RPC? + ************* + + In my opinion, there are several reasons to choose JSON over XML for RPC: + + * Much simpler to read (I suppose this is opinion, but I know I'm right. :) + * Size / Bandwidth - Main reason, a JSON object representation is just much smaller. + * Parsing - JSON should be much quicker to parse than XML. + * Easy class passing with ``jsonclass`` (when enabled) + + In the interest of being fair, there are also a few reasons to choose XML + over JSON: + + * Your server doesn't do JSON (rather obvious) + * Wider XML-RPC support across APIs (can we change this? :)) + * Libraries are more established, i.e. more stable (Let's change this too.) + + Tests + ***** + + Tests are an almost-verbatim drop from the JSON-RPC specification 2.0 page. + They can be run using *unittest* or *nosetest*: + + .. code-block:: console + + python -m unittest discover tests + python3 -m unittest discover tests + nosetests tests + +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.0 +Classifier: Programming Language :: Python :: 3.1 +Classifier: Programming Language :: Python :: 3.2 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/README.rst b/scripts/external_libs/jsonrpclib-pelix-0.2.5/README.rst new file mode 100644 index 00000000..19001933 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/README.rst @@ -0,0 +1,438 @@ +JSONRPClib (patched for Pelix) +############################## + +.. image:: https://pypip.in/license/jsonrpclib-pelix/badge.svg + :target: https://pypi.python.org/pypi/jsonrpclib-pelix/ + +.. image:: https://travis-ci.org/tcalmant/jsonrpclib.svg?branch=master + :target: https://travis-ci.org/tcalmant/jsonrpclib + +.. image:: https://coveralls.io/repos/tcalmant/jsonrpclib/badge.svg?branch=master + :target: https://coveralls.io/r/tcalmant/jsonrpclib?branch=master + + +This library is an implementation of the JSON-RPC specification. +It supports both the original 1.0 specification, as well as the +new (proposed) 2.0 specification, which includes batch submission, keyword +arguments, etc. + +It is licensed under the Apache License, Version 2.0 +(http://www.apache.org/licenses/LICENSE-2.0.html). + + +About this version +****************** + +This is a patched version of the original ``jsonrpclib`` project by +Josh Marshall, available at https://github.com/joshmarshall/jsonrpclib. + +The suffix *-pelix* only indicates that this version works with Pelix Remote +Services, but it is **not** a Pelix specific implementation. + +* This version adds support for Python 3, staying compatible with Python 2. +* It is now possible to use the dispatch_method argument while extending + the SimpleJSONRPCDispatcher, to use a custom dispatcher. + This allows to use this package by Pelix Remote Services. +* It can use thread pools to control the number of threads spawned to handle + notification requests and clients connections. +* The modifications added in other forks of this project have been added: + + * From https://github.com/drdaeman/jsonrpclib: + + * Improved JSON-RPC 1.0 support + * Less strict error response handling + + * From https://github.com/tuomassalo/jsonrpclib: + + * In case of a non-pre-defined error, raise an AppError and give access to + *error.data* + + * From https://github.com/dejw/jsonrpclib: + + * Custom headers can be sent with request and associated tests + +* The support for Unix sockets has been removed, as it is not trivial to convert + to Python 3 (and I don't use them) +* This version cannot be installed with the original ``jsonrpclib``, as it uses + the same package name. + + +Summary +******* + +This library implements the JSON-RPC 2.0 proposed specification in pure Python. +It is designed to be as compatible with the syntax of ``xmlrpclib`` as possible +(it extends where possible), so that projects using ``xmlrpclib`` could easily +be modified to use JSON and experiment with the differences. + +It is backwards-compatible with the 1.0 specification, and supports all of the +new proposed features of 2.0, including: + +* Batch submission (via MultiCall) +* Keyword arguments +* Notifications (both in a batch and 'normal') +* Class translation using the ``__jsonclass__`` key. + +I've added a "SimpleJSONRPCServer", which is intended to emulate the +"SimpleXMLRPCServer" from the default Python distribution. + + +Requirements +************ + +It supports ``cjson`` and ``simplejson``, and looks for the parsers in that +order (searching first for ``cjson``, then for the *built-in* ``json`` in 2.6+, +and then the ``simplejson`` external library). +One of these must be installed to use this library, although if you have a +standard distribution of 2.6+, you should already have one. +Keep in mind that ``cjson`` is supposed to be the quickest, I believe, so if +you are going for full-on optimization you may want to pick it up. + +Since library uses ``contextlib`` module, you should have at least Python 2.5 +installed. + + +Installation +************ + +You can install this from PyPI with one of the following commands (sudo +may be required): + +.. code-block:: console + + easy_install jsonrpclib-pelix + pip install jsonrpclib-pelix + +Alternatively, you can download the source from the GitHub repository +at http://github.com/tcalmant/jsonrpclib and manually install it +with the following commands: + +.. code-block:: console + + git clone git://github.com/tcalmant/jsonrpclib.git + cd jsonrpclib + python setup.py install + + +SimpleJSONRPCServer +******************* + +This is identical in usage (or should be) to the SimpleXMLRPCServer in the +Python standard library. Some of the differences in features are that it +obviously supports notification, batch calls, class translation (if left on), +etc. +Note: The import line is slightly different from the regular SimpleXMLRPCServer, +since the SimpleJSONRPCServer is distributed within the ``jsonrpclib`` library. + +.. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + + server = SimpleJSONRPCServer(('localhost', 8080)) + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_function(lambda x: x, 'ping') + server.serve_forever() + +To start protect the server with SSL, use the following snippet: + +.. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + + # Setup the SSL socket + server = SimpleJSONRPCServer(('localhost', 8080), bind_and_activate=False) + server.socket = ssl.wrap_socket(server.socket, certfile='server.pem', + server_side=True) + server.server_bind() + server.server_activate() + + # ... register functions + # Start the server + server.serve_forever() + + +Notification Thread Pool +======================== + +By default, notification calls are handled in the request handling thread. +It is possible to use a thread pool to handle them, by giving it to the server +using the ``set_notification_pool()`` method: + +.. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + from jsonrpclib.threadpool import ThreadPool + + # Setup the thread pool: between 0 and 10 threads + pool = ThreadPool(max_threads=10, min_threads=0) + + # Don't forget to start it + pool.start() + + # Setup the server + server = SimpleJSONRPCServer(('localhost', 8080), config) + server.set_notification_pool(pool) + + # Register methods + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_function(lambda x: x, 'ping') + + try: + server.serve_forever() + finally: + # Stop the thread pool (let threads finish their current task) + pool.stop() + server.set_notification_pool(None) + + +Threaded server +=============== + +It is also possible to use a thread pool to handle clients requests, using the +``PooledJSONRPCServer`` class. +By default, this class uses pool of 0 to 30 threads. A custom pool can be given +with the ``thread_pool`` parameter of the class constructor. + +The notification pool and the request pool are different: by default, a server +with a request pool doesn't have a notification pool. + +.. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import PooledJSONRPCServer + from jsonrpclib.threadpool import ThreadPool + + # Setup the notification and request pools + nofif_pool = ThreadPool(max_threads=10, min_threads=0) + request_pool = ThreadPool(max_threads=50, min_threads=10) + + # Don't forget to start them + nofif_pool.start() + request_pool.start() + + # Setup the server + server = PooledJSONRPCServer(('localhost', 8080), config, + thread_pool=request_pool) + server.set_notification_pool(nofif_pool) + + # Register methods + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_function(lambda x: x, 'ping') + + try: + server.serve_forever() + finally: + # Stop the thread pools (let threads finish their current task) + request_pool.stop() + nofif_pool.stop() + server.set_notification_pool(None) + +Client Usage +************ + +This is (obviously) taken from a console session. + +.. code-block:: python + + >>> import jsonrpclib + >>> server = jsonrpclib.ServerProxy('http://localhost:8080') + >>> server.add(5,6) + 11 + >>> server.add(x=5, y=10) + 15 + >>> server._notify.add(5,6) + # No result returned... + >>> batch = jsonrpclib.MultiCall(server) + >>> batch.add(5, 6) + >>> batch.ping({'key':'value'}) + >>> batch._notify.add(4, 30) + >>> results = batch() + >>> for result in results: + >>> ... print(result) + 11 + {'key': 'value'} + # Note that there are only two responses -- this is according to spec. + + # Clean up + >>> server('close')() + + # Using client history + >>> history = jsonrpclib.history.History() + >>> server = jsonrpclib.ServerProxy('http://localhost:8080', history=history) + >>> server.add(5,6) + 11 + >>> print(history.request) + {"id": "f682b956-c8e1-4506-9db4-29fe8bc9fcaa", "jsonrpc": "2.0", + "method": "add", "params": [5, 6]} + >>> print(history.response) + {"id": "f682b956-c8e1-4506-9db4-29fe8bc9fcaa", "jsonrpc": "2.0", + "result": 11} + + # Clean up + >>> server('close')() + +If you need 1.0 functionality, there are a bunch of places you can pass that in, +although the best is just to give a specific configuration to +``jsonrpclib.ServerProxy``: + +.. code-block:: python + + >>> import jsonrpclib + >>> jsonrpclib.config.DEFAULT.version + 2.0 + >>> config = jsonrpclib.config.Config(version=1.0) + >>> history = jsonrpclib.history.History() + >>> server = jsonrpclib.ServerProxy('http://localhost:8080', config=config, + history=history) + >>> server.add(7, 10) + 17 + >>> print(history.request) + {"id": "827b2923-5b37-49a5-8b36-e73920a16d32", + "method": "add", "params": [7, 10]} + >>> print(history.response) + {"id": "827b2923-5b37-49a5-8b36-e73920a16d32", "error": null, "result": 17} + >>> server('close')() + +The equivalent ``loads`` and ``dumps`` functions also exist, although with minor +modifications. The ``dumps`` arguments are almost identical, but it adds three +arguments: ``rpcid`` for the 'id' key, ``version`` to specify the JSON-RPC +compatibility, and ``notify`` if it's a request that you want to be a +notification. + +Additionally, the ``loads`` method does not return the params and method like +``xmlrpclib``, but instead a.) parses for errors, raising ProtocolErrors, and +b.) returns the entire structure of the request / response for manual parsing. + + +Additional headers +****************** + +If your remote service requires custom headers in request, you can pass them +as as a ``headers`` keyword argument, when creating the ``ServerProxy``: + +.. code-block:: python + + >>> import jsonrpclib + >>> server = jsonrpclib.ServerProxy("http://localhost:8080", + headers={'X-Test' : 'Test'}) + +You can also put additional request headers only for certain method invocation: + +.. code-block:: python + + >>> import jsonrpclib + >>> server = jsonrpclib.Server("http://localhost:8080") + >>> with server._additional_headers({'X-Test' : 'Test'}) as test_server: + ... test_server.ping(42) + ... + >>> # X-Test header will be no longer sent in requests + +Of course ``_additional_headers`` contexts can be nested as well. + + +Class Translation +***************** + +I've recently added "automatic" class translation support, although it is +turned off by default. This can be devastatingly slow if improperly used, so +the following is just a short list of things to keep in mind when using it. + +* Keep It (the object) Simple Stupid. (for exceptions, keep reading.) +* Do not require init params (for exceptions, keep reading) +* Getter properties without setters could be dangerous (read: not tested) + +If any of the above are issues, use the _serialize method. (see usage below) +The server and client must BOTH have use_jsonclass configuration item on and +they must both have access to the same libraries used by the objects for +this to work. + +If you have excessively nested arguments, it would be better to turn off the +translation and manually invoke it on specific objects using +``jsonrpclib.jsonclass.dump`` / ``jsonrpclib.jsonclass.load`` (since the default +behavior recursively goes through attributes and lists / dicts / tuples). + + Sample file: *test_obj.py* + +.. code-block:: python + + # This object is /very/ simple, and the system will look through the + # attributes and serialize what it can. + class TestObj(object): + foo = 'bar' + + # This object requires __init__ params, so it uses the _serialize method + # and returns a tuple of init params and attribute values (the init params + # can be a dict or a list, but the attribute values must be a dict.) + class TestSerial(object): + foo = 'bar' + def __init__(self, *args): + self.args = args + def _serialize(self): + return (self.args, {'foo':self.foo,}) + +* Sample usage + +.. code-block:: python + + >>> import jsonrpclib + >>> import test_obj + + # History is used only to print the serialized form of beans + >>> history = jsonrpclib.history.History() + >>> testobj1 = test_obj.TestObj() + >>> testobj2 = test_obj.TestSerial() + >>> server = jsonrpclib.Server('http://localhost:8080', history=history) + + # The 'ping' just returns whatever is sent + >>> ping1 = server.ping(testobj1) + >>> ping2 = server.ping(testobj2) + + >>> print(history.request) + {"id": "7805f1f9-9abd-49c6-81dc-dbd47229fe13", "jsonrpc": "2.0", + "method": "ping", "params": [{"__jsonclass__": + ["test_obj.TestSerial", []], "foo": "bar"} + ]} + >>> print(history.response) + {"id": "7805f1f9-9abd-49c6-81dc-dbd47229fe13", "jsonrpc": "2.0", + "result": {"__jsonclass__": ["test_obj.TestSerial", []], "foo": "bar"}} + +This behavior is turned by default. To deactivate it, just set the +``use_jsonclass`` member of a server ``Config`` to False. +If you want to use a per-class serialization method, set its name in the +``serialize_method`` member of a server ``Config``. +Finally, if you are using classes that you have defined in the implementation +(as in, not a separate library), you'll need to add those (on BOTH the server +and the client) using the ``config.classes.add()`` method. + +Feedback on this "feature" is very, VERY much appreciated. + +Why JSON-RPC? +************* + +In my opinion, there are several reasons to choose JSON over XML for RPC: + +* Much simpler to read (I suppose this is opinion, but I know I'm right. :) +* Size / Bandwidth - Main reason, a JSON object representation is just much smaller. +* Parsing - JSON should be much quicker to parse than XML. +* Easy class passing with ``jsonclass`` (when enabled) + +In the interest of being fair, there are also a few reasons to choose XML +over JSON: + +* Your server doesn't do JSON (rather obvious) +* Wider XML-RPC support across APIs (can we change this? :)) +* Libraries are more established, i.e. more stable (Let's change this too.) + +Tests +***** + +Tests are an almost-verbatim drop from the JSON-RPC specification 2.0 page. +They can be run using *unittest* or *nosetest*: + +.. code-block:: console + + python -m unittest discover tests + python3 -m unittest discover tests + nosetests tests diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/SimpleJSONRPCServer.py b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/SimpleJSONRPCServer.py new file mode 100644 index 00000000..e9fe4e68 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/SimpleJSONRPCServer.py @@ -0,0 +1,602 @@ +#!/usr/bin/python +# -- Content-Encoding: UTF-8 -- +""" +Defines a request dispatcher, a HTTP request handler, a HTTP server and a +CGI request handler. + +:authors: Josh Marshall, Thomas Calmant +:copyright: Copyright 2015, isandlaTech +:license: Apache License 2.0 +:version: 0.2.5 + +.. + + Copyright 2015 isandlaTech + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +# Module version +__version_info__ = (0, 2, 5) +__version__ = ".".join(str(x) for x in __version_info__) + +# Documentation strings format +__docformat__ = "restructuredtext en" + +# ------------------------------------------------------------------------------ +# Local modules +from jsonrpclib import Fault +import jsonrpclib.config +import jsonrpclib.utils as utils +import jsonrpclib.threadpool + +# Standard library +import logging +import socket +import sys +import traceback + +# Prepare the logger +_logger = logging.getLogger(__name__) + +try: + # Python 3 + # pylint: disable=F0401,E0611 + import xmlrpc.server as xmlrpcserver + import socketserver +except (ImportError, AttributeError): + # Python 2 or IronPython + # pylint: disable=F0401,E0611 + import SimpleXMLRPCServer as xmlrpcserver + import SocketServer as socketserver + +try: + # Windows + import fcntl +except ImportError: + # Other systems + # pylint: disable=C0103 + fcntl = None + +# ------------------------------------------------------------------------------ + + +def get_version(request): + """ + Computes the JSON-RPC version + + :param request: A request dictionary + :return: The JSON-RPC version or None + """ + if 'jsonrpc' in request: + return 2.0 + elif 'id' in request: + return 1.0 + + return None + + +def validate_request(request, json_config): + """ + Validates the format of a request dictionary + + :param request: A request dictionary + :param json_config: A JSONRPClib Config instance + :return: True if the dictionary is valid, else a Fault object + """ + if not isinstance(request, utils.DictType): + # Invalid request type + fault = Fault(-32600, 'Request must be a dict, not {0}' + .format(type(request).__name__), + config=json_config) + _logger.warning("Invalid request content: %s", fault) + return fault + + # Get the request ID + rpcid = request.get('id', None) + + # Check request version + version = get_version(request) + if not version: + fault = Fault(-32600, 'Request {0} invalid.'.format(request), + rpcid=rpcid, config=json_config) + _logger.warning("No version in request: %s", fault) + return fault + + # Default parameters: empty list + request.setdefault('params', []) + + # Check parameters + method = request.get('method', None) + params = request.get('params') + param_types = (utils.ListType, utils.DictType, utils.TupleType) + + if not method or not isinstance(method, utils.string_types) or \ + not isinstance(params, param_types): + # Invalid type of method name or parameters + fault = Fault(-32600, 'Invalid request parameters or method.', + rpcid=rpcid, config=json_config) + _logger.warning("Invalid request content: %s", fault) + return fault + + # Valid request + return True + +# ------------------------------------------------------------------------------ + + +class NoMulticallResult(Exception): + """ + No result in multicall + """ + pass + + +class SimpleJSONRPCDispatcher(xmlrpcserver.SimpleXMLRPCDispatcher, object): + """ + Mix-in class that dispatches JSON-RPC requests. + + This class is used to register JSON-RPC method handlers + and then to dispatch them. This class doesn't need to be + instanced directly when used by SimpleJSONRPCServer. + """ + def __init__(self, encoding=None, config=jsonrpclib.config.DEFAULT): + """ + Sets up the dispatcher with the given encoding. + None values are allowed. + """ + xmlrpcserver.SimpleXMLRPCDispatcher.__init__( + self, allow_none=True, encoding=encoding or "UTF-8") + self.json_config = config + + # Notification thread pool + self.__notification_pool = None + + def set_notification_pool(self, thread_pool): + """ + Sets the thread pool to use to handle notifications + """ + self.__notification_pool = thread_pool + + def _unmarshaled_dispatch(self, request, dispatch_method=None): + """ + Loads the request dictionary (unmarshaled), calls the method(s) + accordingly and returns a JSON-RPC dictionary (not marshaled) + + :param request: JSON-RPC request dictionary (or list of) + :param dispatch_method: Custom dispatch method (for method resolution) + :return: A JSON-RPC dictionary (or an array of) or None if the request + was a notification + :raise NoMulticallResult: No result in batch + """ + if not request: + # Invalid request dictionary + fault = Fault(-32600, 'Request invalid -- no request data.', + config=self.json_config) + _logger.warning("Invalid request: %s", fault) + return fault.dump() + + if isinstance(request, utils.ListType): + # This SHOULD be a batch, by spec + responses = [] + for req_entry in request: + # Validate the request + result = validate_request(req_entry, self.json_config) + if isinstance(result, Fault): + responses.append(result.dump()) + continue + + # Call the method + resp_entry = self._marshaled_single_dispatch(req_entry, + dispatch_method) + + # Store its result + if isinstance(resp_entry, Fault): + # pylint: disable=E1103 + responses.append(resp_entry.dump()) + elif resp_entry is not None: + responses.append(resp_entry) + + if not responses: + # No non-None result + _logger.error("No result in Multicall") + raise NoMulticallResult("No result") + + return responses + + else: + # Single call + result = validate_request(request, self.json_config) + if isinstance(result, Fault): + return result.dump() + + # Call the method + response = self._marshaled_single_dispatch(request, + dispatch_method) + if isinstance(response, Fault): + # pylint: disable=E1103 + return response.dump() + + return response + + def _marshaled_dispatch(self, data, dispatch_method=None, path=None): + """ + Parses the request data (marshaled), calls method(s) and returns a + JSON string (marshaled) + + :param data: A JSON request string + :param dispatch_method: Custom dispatch method (for method resolution) + :param path: Unused parameter, to keep compatibility with xmlrpclib + :return: A JSON-RPC response string (marshaled) + """ + # Parse the request + try: + request = jsonrpclib.loads(data, self.json_config) + except Exception as ex: + # Parsing/loading error + fault = Fault(-32700, 'Request {0} invalid. ({1}:{2})' + .format(data, type(ex).__name__, ex), + config=self.json_config) + _logger.warning("Error parsing request: %s", fault) + return fault.response() + + # Get the response dictionary + try: + response = self._unmarshaled_dispatch(request, dispatch_method) + if response is not None: + # Compute the string representation of the dictionary/list + return jsonrpclib.jdumps(response, self.encoding) + else: + # No result (notification) + return '' + except NoMulticallResult: + # Return an empty string (jsonrpclib internal behaviour) + return '' + + def _marshaled_single_dispatch(self, request, dispatch_method=None): + """ + Dispatches a single method call + + :param request: A validated request dictionary + :param dispatch_method: Custom dispatch method (for method resolution) + :return: A JSON-RPC response dictionary, or None if it was a + notification request + """ + method = request.get('method') + params = request.get('params') + + # Prepare a request-specific configuration + if 'jsonrpc' not in request and self.json_config.version >= 2: + # JSON-RPC 1.0 request on a JSON-RPC 2.0 + # => compatibility needed + config = self.json_config.copy() + config.version = 1.0 + else: + # Keep server configuration as is + config = self.json_config + + # Test if this is a notification request + is_notification = 'id' not in request or request['id'] in (None, '') + if is_notification and self.__notification_pool is not None: + # Use the thread pool for notifications + if dispatch_method is not None: + self.__notification_pool.enqueue(dispatch_method, + method, params) + else: + self.__notification_pool.enqueue(self._dispatch, + method, params, config) + + # Return immediately + return None + else: + # Synchronous call + try: + # Call the method + if dispatch_method is not None: + response = dispatch_method(method, params) + else: + response = self._dispatch(method, params, config) + except Exception as ex: + # Return a fault + fault = Fault(-32603, '{0}:{1}'.format(type(ex).__name__, ex), + config=config) + _logger.error("Error calling method %s: %s", method, fault) + return fault.dump() + + if is_notification: + # It's a notification, no result needed + # Do not use 'not id' as it might be the integer 0 + return None + + # Prepare a JSON-RPC dictionary + try: + return jsonrpclib.dump(response, rpcid=request['id'], + is_response=True, config=config) + except Exception as ex: + # JSON conversion exception + fault = Fault(-32603, '{0}:{1}'.format(type(ex).__name__, ex), + config=config) + _logger.error("Error preparing JSON-RPC result: %s", fault) + return fault.dump() + + def _dispatch(self, method, params, config=None): + """ + Default method resolver and caller + + :param method: Name of the method to call + :param params: List of arguments to give to the method + :param config: Request-specific configuration + :return: The result of the method + """ + config = config or self.json_config + + func = None + try: + # Look into registered methods + func = self.funcs[method] + except KeyError: + if self.instance is not None: + # Try with the registered instance + try: + # Instance has a custom dispatcher + return getattr(self.instance, '_dispatch')(method, params) + except AttributeError: + # Resolve the method name in the instance + try: + func = xmlrpcserver.resolve_dotted_attribute( + self.instance, method, True) + except AttributeError: + # Unknown method + pass + + if func is not None: + try: + # Call the method + if isinstance(params, utils.ListType): + return func(*params) + else: + return func(**params) + except TypeError as ex: + # Maybe the parameters are wrong + fault = Fault(-32602, 'Invalid parameters: {0}'.format(ex), + config=config) + _logger.warning("Invalid call parameters: %s", fault) + return fault + except: + # Method exception + err_lines = traceback.format_exc().splitlines() + trace_string = '{0} | {1}'.format(err_lines[-3], err_lines[-1]) + fault = Fault(-32603, 'Server error: {0}'.format(trace_string), + config=config) + _logger.exception("Server-side exception: %s", fault) + return fault + else: + # Unknown method + fault = Fault(-32601, 'Method {0} not supported.'.format(method), + config=config) + _logger.warning("Unknown method: %s", fault) + return fault + +# ------------------------------------------------------------------------------ + + +class SimpleJSONRPCRequestHandler(xmlrpcserver.SimpleXMLRPCRequestHandler): + """ + HTTP request handler. + + The server that receives the requests must have a json_config member, + containing a JSONRPClib Config instance + """ + def do_POST(self): + """ + Handles POST requests + """ + if not self.is_rpc_path_valid(): + self.report_404() + return + + # Retrieve the configuration + config = getattr(self.server, 'json_config', jsonrpclib.config.DEFAULT) + + try: + # Read the request body + max_chunk_size = 10 * 1024 * 1024 + size_remaining = int(self.headers["content-length"]) + chunks = [] + while size_remaining: + chunk_size = min(size_remaining, max_chunk_size) + raw_chunk = self.rfile.read(chunk_size) + if not raw_chunk: + break + chunks.append(utils.from_bytes(raw_chunk)) + size_remaining -= len(chunks[-1]) + data = ''.join(chunks) + + try: + # Decode content + data = self.decode_request_content(data) + if data is None: + # Unknown encoding, response has been sent + return + except AttributeError: + # Available since Python 2.7 + pass + + # Execute the method + response = self.server._marshaled_dispatch( + data, getattr(self, '_dispatch', None), self.path) + + # No exception: send a 200 OK + self.send_response(200) + except: + # Exception: send 500 Server Error + self.send_response(500) + err_lines = traceback.format_exc().splitlines() + trace_string = '{0} | {1}'.format(err_lines[-3], err_lines[-1]) + fault = jsonrpclib.Fault(-32603, 'Server error: {0}' + .format(trace_string), config=config) + _logger.exception("Server-side error: %s", fault) + response = fault.response() + + if response is None: + # Avoid to send None + response = '' + + # Convert the response to the valid string format + response = utils.to_bytes(response) + + # Send it + self.send_header("Content-type", config.content_type) + self.send_header("Content-length", str(len(response))) + self.end_headers() + if response: + self.wfile.write(response) + +# ------------------------------------------------------------------------------ + + +class SimpleJSONRPCServer(socketserver.TCPServer, SimpleJSONRPCDispatcher): + """ + JSON-RPC server (and dispatcher) + """ + # This simplifies server restart after error + allow_reuse_address = True + + # pylint: disable=C0103 + def __init__(self, addr, requestHandler=SimpleJSONRPCRequestHandler, + logRequests=True, encoding=None, bind_and_activate=True, + address_family=socket.AF_INET, + config=jsonrpclib.config.DEFAULT): + """ + Sets up the server and the dispatcher + + :param addr: The server listening address + :param requestHandler: Custom request handler + :param logRequests: Flag to(de)activate requests logging + :param encoding: The dispatcher request encoding + :param bind_and_activate: If True, starts the server immediately + :param address_family: The server listening address family + :param config: A JSONRPClib Config instance + """ + # Set up the dispatcher fields + SimpleJSONRPCDispatcher.__init__(self, encoding, config) + + # Prepare the server configuration + # logRequests is used by SimpleXMLRPCRequestHandler + self.logRequests = logRequests + self.address_family = address_family + self.json_config = config + + # Work on the request handler + class RequestHandlerWrapper(requestHandler, object): + """ + Wraps the request handle to have access to the configuration + """ + def __init__(self, *args, **kwargs): + """ + Constructs the wrapper after having stored the configuration + """ + self.config = config + super(RequestHandlerWrapper, self).__init__(*args, **kwargs) + + # Set up the server + socketserver.TCPServer.__init__(self, addr, requestHandler, + bind_and_activate) + + # Windows-specific + if fcntl is not None and hasattr(fcntl, 'FD_CLOEXEC'): + flags = fcntl.fcntl(self.fileno(), fcntl.F_GETFD) + flags |= fcntl.FD_CLOEXEC + fcntl.fcntl(self.fileno(), fcntl.F_SETFD, flags) + +# ------------------------------------------------------------------------------ + + +class PooledJSONRPCServer(SimpleJSONRPCServer, socketserver.ThreadingMixIn): + """ + JSON-RPC server based on a thread pool + """ + def __init__(self, addr, requestHandler=SimpleJSONRPCRequestHandler, + logRequests=True, encoding=None, bind_and_activate=True, + address_family=socket.AF_INET, + config=jsonrpclib.config.DEFAULT, thread_pool=None): + """ + Sets up the server and the dispatcher + + :param addr: The server listening address + :param requestHandler: Custom request handler + :param logRequests: Flag to(de)activate requests logging + :param encoding: The dispatcher request encoding + :param bind_and_activate: If True, starts the server immediately + :param address_family: The server listening address family + :param config: A JSONRPClib Config instance + :param thread_pool: A ThreadPool object. The pool must be started. + """ + # Normalize the thread pool + if thread_pool is None: + # Start a thread pool with 30 threads max, 0 thread min + thread_pool = jsonrpclib.threadpool.ThreadPool( + 30, 0, logname="PooledJSONRPCServer") + thread_pool.start() + + # Store the thread pool + self.__request_pool = thread_pool + + # Prepare the server + SimpleJSONRPCServer.__init__(self, addr, requestHandler, logRequests, + encoding, bind_and_activate, + address_family, config) + + def process_request(self, request, client_address): + """ + Handle a client request: queue it in the thread pool + """ + self.__request_pool.enqueue(self.process_request_thread, + request, client_address) + + def server_close(self): + """ + Clean up the server + """ + SimpleJSONRPCServer.server_close(self) + self.__request_pool.stop() + +# ------------------------------------------------------------------------------ + + +class CGIJSONRPCRequestHandler(SimpleJSONRPCDispatcher): + """ + JSON-RPC CGI handler (and dispatcher) + """ + def __init__(self, encoding=None, config=jsonrpclib.config.DEFAULT): + """ + Sets up the dispatcher + + :param encoding: Dispatcher encoding + :param config: A JSONRPClib Config instance + """ + SimpleJSONRPCDispatcher.__init__(self, encoding, config) + + def handle_jsonrpc(self, request_text): + """ + Handle a JSON-RPC request + """ + response = self._marshaled_dispatch(request_text) + sys.stdout.write('Content-Type: {0}\r\n' + .format(self.json_config.content_type)) + sys.stdout.write('Content-Length: {0:d}\r\n'.format(len(response))) + sys.stdout.write('\r\n') + sys.stdout.write(response) + + # XML-RPC alias + handle_xmlrpc = handle_jsonrpc diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/__init__.py b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/__init__.py new file mode 100644 index 00000000..a92774ab --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/__init__.py @@ -0,0 +1,34 @@ +#!/usr/bin/python +# -- Content-Encoding: UTF-8 -- +""" +Aliases to ease access to jsonrpclib classes + +:authors: Josh Marshall, Thomas Calmant +:copyright: Copyright 2015, isandlaTech +:license: Apache License 2.0 +:version: 0.2.5 + +.. + + Copyright 2015 isandlaTech + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +# Easy access to utility methods and classes +from jsonrpclib.jsonrpc import Server, ServerProxy +from jsonrpclib.jsonrpc import MultiCall, Fault, ProtocolError, AppError +from jsonrpclib.jsonrpc import loads, dumps, load, dump +from jsonrpclib.jsonrpc import jloads, jdumps +import jsonrpclib.history as history +import jsonrpclib.utils as utils diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/config.py b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/config.py new file mode 100644 index 00000000..77838d4e --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/config.py @@ -0,0 +1,141 @@ +#!/usr/bin/python +# -- Content-Encoding: UTF-8 -- +""" +The configuration module. + +:copyright: Copyright 2015, isandlaTech +:license: Apache License 2.0 +:version: 0.2.5 + +.. + + Copyright 2015 isandlaTech + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +# Module version +__version_info__ = (0, 2, 5) +__version__ = ".".join(str(x) for x in __version_info__) + +# Documentation strings format +__docformat__ = "restructuredtext en" + +# ------------------------------------------------------------------------------ + +import sys + +# ------------------------------------------------------------------------------ + + +class LocalClasses(dict): + """ + Associates local classes with their names (used in the jsonclass module) + """ + def add(self, cls, name=None): + """ + Stores a local class + + :param cls: A class + :param name: Custom name used in the __jsonclass__ attribute + """ + if not name: + name = cls.__name__ + self[name] = cls + +# ------------------------------------------------------------------------------ + + +class Config(object): + """ + This is pretty much used exclusively for the 'jsonclass' + functionality... set use_jsonclass to False to turn it off. + You can change serialize_method and ignore_attribute, or use + the local_classes.add(class) to include "local" classes. + """ + def __init__(self, version=2.0, content_type="application/json-rpc", + user_agent=None, use_jsonclass=True, + serialize_method='_serialize', + ignore_attribute='_ignore', + serialize_handlers=None): + """ + Sets up a configuration of JSONRPClib + + :param version: JSON-RPC specification version + :param content_type: HTTP content type header value + :param user_agent: The HTTP request user agent + :param use_jsonclass: Allow bean marshalling + :param serialize_method: A string that references the method on a + custom class object which is responsible for + returning a tuple of the arguments and a dict + of attributes. + :param ignore_attribute: A string that references the attribute on a + custom class object which holds strings and/or + references of the attributes the class + translator should ignore. + :param serialize_handlers: A dictionary of dump handler functions by + type for additional type support and for + overriding dump of built-in types in utils + """ + # JSON-RPC specification + self.version = version + + # Change to False to keep __jsonclass__ entries raw. + self.use_jsonclass = use_jsonclass + + # it SHOULD be 'application/json-rpc' + # but MAY be 'application/json' or 'application/jsonrequest' + self.content_type = content_type + + # Default user agent + if user_agent is None: + user_agent = 'jsonrpclib/{0} (Python {1})'.format( + __version__, '.'.join(str(ver) + for ver in sys.version_info[0:3])) + self.user_agent = user_agent + + # The list of classes to use for jsonclass translation. + self.classes = LocalClasses() + + # The serialize_method should be a string that references the + # method on a custom class object which is responsible for + # returning a tuple of the constructor arguments and a dict of + # attributes. + self.serialize_method = serialize_method + + # The ignore attribute should be a string that references the + # attribute on a custom class object which holds strings and / or + # references of the attributes the class translator should ignore. + self.ignore_attribute = ignore_attribute + + # The list of serialize handler functions for jsonclass dump. + # Used for handling additional types and overriding built-in types. + # Functions are expected to have the same parameters as jsonclass dump + # (possibility to call standard jsonclass dump function within). + self.serialize_handlers = serialize_handlers or {} + + def copy(self): + """ + Returns a shallow copy of this configuration bean + + :return: A shallow copy of this configuration + """ + new_config = Config(self.version, self.content_type, self.user_agent, + self.use_jsonclass, self.serialize_method, + self.ignore_attribute, None) + new_config.classes = self.classes.copy() + new_config.serialize_handlers = self.serialize_handlers.copy() + return new_config + +# Default configuration +DEFAULT = Config() diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/history.py b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/history.py new file mode 100644 index 00000000..288d9539 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/history.py @@ -0,0 +1,95 @@ +#!/usr/bin/python +# -- Content-Encoding: UTF-8 -- +""" +The history module. + +:authors: Josh Marshall, Thomas Calmant +:copyright: Copyright 2015, isandlaTech +:license: Apache License 2.0 +:version: 0.2.5 + +.. + + Copyright 2015 isandlaTech + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +# Module version +__version_info__ = (0, 2, 5) +__version__ = ".".join(str(x) for x in __version_info__) + +# Documentation strings format +__docformat__ = "restructuredtext en" + +# ------------------------------------------------------------------------------ + + +class History(object): + """ + This holds all the response and request objects for a + session. A server using this should call "clear" after + each request cycle in order to keep it from clogging + memory. + """ + def __init__(self): + """ + Sets up members + """ + self.requests = [] + self.responses = [] + + def add_response(self, response_obj): + """ + Adds a response to the history + + :param response_obj: Response content + """ + self.responses.append(response_obj) + + def add_request(self, request_obj): + """ + Adds a request to the history + + :param request_obj: A request object + """ + self.requests.append(request_obj) + + @property + def request(self): + """ + Returns the latest stored request or None + """ + try: + return self.requests[-1] + + except IndexError: + return None + + @property + def response(self): + """ + Returns the latest stored response or None + """ + try: + return self.responses[-1] + + except IndexError: + return None + + def clear(self): + """ + Clears the history lists + """ + del self.requests[:] + del self.responses[:] diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/jsonclass.py b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/jsonclass.py new file mode 100644 index 00000000..6bcbeab7 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/jsonclass.py @@ -0,0 +1,295 @@ +#!/usr/bin/python +# -- Content-Encoding: UTF-8 -- +""" +The serialization module + +:authors: Josh Marshall, Thomas Calmant +:copyright: Copyright 2015, isandlaTech +:license: Apache License 2.0 +:version: 0.2.5 + +.. + + Copyright 2015 isandlaTech + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +# Module version +__version_info__ = (0, 2, 5) +__version__ = ".".join(str(x) for x in __version_info__) + +# Documentation strings format +__docformat__ = "restructuredtext en" + +# ------------------------------------------------------------------------------ + +# Local package +import jsonrpclib.config +import jsonrpclib.utils as utils + +# Standard library +import inspect +import re + +# ------------------------------------------------------------------------------ + +# Supported transmitted code +SUPPORTED_TYPES = (utils.DictType,) + utils.iterable_types \ + + utils.primitive_types + +# Regex of invalid module characters +INVALID_MODULE_CHARS = r'[^a-zA-Z0-9\_\.]' + +# ------------------------------------------------------------------------------ + + +class TranslationError(Exception): + """ + Unmarshaling exception + """ + pass + + +def _slots_finder(clazz, fields_set): + """ + Recursively visits the class hierarchy to find all slots + + :param clazz: Class to analyze + :param fields_set: Set where to store __slots___ content + """ + # ... class level + try: + fields_set.update(clazz.__slots__) + except AttributeError: + pass + + # ... parent classes level + for base_class in clazz.__bases__: + _slots_finder(base_class, fields_set) + + +def _find_fields(obj): + """ + Returns the names of the fields of the given object + + :param obj: An object to analyze + :return: A set of field names + """ + # Find fields... + fields = set() + + # ... using __dict__ + try: + fields.update(obj.__dict__) + except AttributeError: + pass + + # ... using __slots__ + _slots_finder(obj.__class__, fields) + return fields + + +def dump(obj, serialize_method=None, ignore_attribute=None, ignore=None, + config=jsonrpclib.config.DEFAULT): + """ + Transforms the given object into a JSON-RPC compliant form. + Converts beans into dictionaries with a __jsonclass__ entry. + Doesn't change primitive types. + + :param obj: An object to convert + :param serialize_method: Custom serialization method + :param ignore_attribute: Name of the object attribute containing the names + of members to ignore + :param ignore: A list of members to ignore + :param config: A JSONRPClib Config instance + :return: A JSON-RPC compliant object + """ + # Normalize arguments + serialize_method = serialize_method or config.serialize_method + ignore_attribute = ignore_attribute or config.ignore_attribute + ignore = ignore or [] + + # Parse / return default "types"... + # Apply additional types, override built-in types + # (reminder: config.serialize_handlers is a dict) + try: + serializer = config.serialize_handlers[type(obj)] + except KeyError: + # Not a serializer + pass + else: + if serializer is not None: + return serializer(obj, serialize_method, ignore_attribute, + ignore, config) + + # Primitive + if isinstance(obj, utils.primitive_types): + return obj + + # Iterative + elif isinstance(obj, utils.iterable_types): + # List, set or tuple + return [dump(item, serialize_method, ignore_attribute, ignore, config) + for item in obj] + + elif isinstance(obj, utils.DictType): + # Dictionary + return dict((key, dump(value, serialize_method, + ignore_attribute, ignore, config)) + for key, value in obj.items()) + + # It's not a standard type, so it needs __jsonclass__ + module_name = inspect.getmodule(type(obj)).__name__ + json_class = obj.__class__.__name__ + + if module_name not in ('', '__main__'): + json_class = '{0}.{1}'.format(module_name, json_class) + + # Keep the class name in the returned object + return_obj = {"__jsonclass__": [json_class]} + + # If a serialization method is defined.. + if hasattr(obj, serialize_method): + # Params can be a dict (keyword) or list (positional) + # Attrs MUST be a dict. + serialize = getattr(obj, serialize_method) + params, attrs = serialize() + return_obj['__jsonclass__'].append(params) + return_obj.update(attrs) + return return_obj + + else: + # Otherwise, try to figure it out + # Obviously, we can't assume to know anything about the + # parameters passed to __init__ + return_obj['__jsonclass__'].append([]) + + # Prepare filtering lists + known_types = SUPPORTED_TYPES + tuple(config.serialize_handlers) + ignore_list = getattr(obj, ignore_attribute, []) + ignore + + # Find fields and filter them by name + fields = _find_fields(obj) + fields.difference_update(ignore_list) + + # Dump field values + attrs = {} + for attr_name in fields: + attr_value = getattr(obj, attr_name) + if isinstance(attr_value, known_types) and \ + attr_value not in ignore_list: + attrs[attr_name] = dump(attr_value, serialize_method, + ignore_attribute, ignore, config) + return_obj.update(attrs) + return return_obj + +# ------------------------------------------------------------------------------ + + +def load(obj, classes=None): + """ + If 'obj' is a dictionary containing a __jsonclass__ entry, converts the + dictionary item into a bean of this class. + + :param obj: An object from a JSON-RPC dictionary + :param classes: A custom {name: class} dictionary + :return: The loaded object + """ + # Primitive + if isinstance(obj, utils.primitive_types): + return obj + + # List, set or tuple + elif isinstance(obj, utils.iterable_types): + # This comes from a JSON parser, so it can only be a list... + return [load(entry) for entry in obj] + + # Otherwise, it's a dict type + elif '__jsonclass__' not in obj: + return dict((key, load(value)) for key, value in obj.items()) + + # It's a dictionary, and it has a __jsonclass__ + orig_module_name = obj['__jsonclass__'][0] + params = obj['__jsonclass__'][1] + + # Validate the module name + if not orig_module_name: + raise TranslationError('Module name empty.') + + json_module_clean = re.sub(INVALID_MODULE_CHARS, '', orig_module_name) + if json_module_clean != orig_module_name: + raise TranslationError('Module name {0} has invalid characters.' + .format(orig_module_name)) + + # Load the class + json_module_parts = json_module_clean.split('.') + json_class = None + if classes and len(json_module_parts) == 1: + # Local class name -- probably means it won't work + try: + json_class = classes[json_module_parts[0]] + except KeyError: + raise TranslationError('Unknown class or module {0}.' + .format(json_module_parts[0])) + + else: + # Module + class + json_class_name = json_module_parts.pop() + json_module_tree = '.'.join(json_module_parts) + try: + # Use fromlist to load the module itself, not the package + temp_module = __import__(json_module_tree, + fromlist=[json_class_name]) + except ImportError: + raise TranslationError('Could not import {0} from module {1}.' + .format(json_class_name, json_module_tree)) + + try: + json_class = getattr(temp_module, json_class_name) + except AttributeError: + raise TranslationError("Unknown class {0}.{1}." + .format(json_module_tree, json_class_name)) + + # Create the object + new_obj = None + if isinstance(params, utils.ListType): + try: + new_obj = json_class(*params) + except TypeError as ex: + raise TranslationError("Error instantiating {0}: {1}" + .format(json_class.__name__, ex)) + + elif isinstance(params, utils.DictType): + try: + new_obj = json_class(**params) + except TypeError as ex: + raise TranslationError("Error instantiating {0}: {1}" + .format(json_class.__name__, ex)) + + else: + raise TranslationError("Constructor args must be a dict or a list, " + "not {0}".format(type(params).__name__)) + + # Remove the class information, as it must be ignored during the + # reconstruction of the object + raw_jsonclass = obj.pop('__jsonclass__') + + for key, value in obj.items(): + # Recursive loading + setattr(new_obj, key, load(value, classes)) + + # Restore the class information for further usage + obj['__jsonclass__'] = raw_jsonclass + + return new_obj diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/jsonrpc.py b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/jsonrpc.py new file mode 100644 index 00000000..8ea3a9c8 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/jsonrpc.py @@ -0,0 +1,1192 @@ +#!/usr/bin/python +# -- Content-Encoding: UTF-8 -- +""" +============================ +JSONRPC Library (jsonrpclib) +============================ + +This library is a JSON-RPC v.2 (proposed) implementation which +follows the xmlrpclib API for portability between clients. It +uses the same Server / ServerProxy, loads, dumps, etc. syntax, +while providing features not present in XML-RPC like: + +* Keyword arguments +* Notifications +* Versioning +* Batches and batch notifications + +Eventually, I'll add a SimpleXMLRPCServer compatible library, +and other things to tie the thing off nicely. :) + +For a quick-start, just open a console and type the following, +replacing the server address, method, and parameters +appropriately. +>>> import jsonrpclib +>>> server = jsonrpclib.Server('http://localhost:8181') +>>> server.add(5, 6) +11 +>>> server._notify.add(5, 6) +>>> batch = jsonrpclib.MultiCall(server) +>>> batch.add(3, 50) +>>> batch.add(2, 3) +>>> batch._notify.add(3, 5) +>>> batch() +[53, 5] + +See https://github.com/tcalmant/jsonrpclib for more info. + +:authors: Josh Marshall, Thomas Calmant +:copyright: Copyright 2015, isandlaTech +:license: Apache License 2.0 +:version: 0.2.5 + +.. + + Copyright 2015 isandlaTech + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +# Module version +__version_info__ = (0, 2, 5) +__version__ = ".".join(str(x) for x in __version_info__) + +# Documentation strings format +__docformat__ = "restructuredtext en" + +# ------------------------------------------------------------------------------ + +# Library includes +import jsonrpclib.config +import jsonrpclib.utils as utils + +# Standard library +import contextlib +import logging +import sys +import uuid + +# Create the logger +_logger = logging.getLogger(__name__) + +try: + # Python 3 + # pylint: disable=F0401,E0611 + from urllib.parse import splittype + from urllib.parse import splithost + from xmlrpc.client import Transport as XMLTransport + from xmlrpc.client import SafeTransport as XMLSafeTransport + from xmlrpc.client import ServerProxy as XMLServerProxy + from xmlrpc.client import _Method as XML_Method + +except ImportError: + # Python 2 + # pylint: disable=F0401,E0611 + from urllib import splittype + from urllib import splithost + from xmlrpclib import Transport as XMLTransport + from xmlrpclib import SafeTransport as XMLSafeTransport + from xmlrpclib import ServerProxy as XMLServerProxy + from xmlrpclib import _Method as XML_Method + +# ------------------------------------------------------------------------------ +# JSON library import + +# JSON class serialization +from jsonrpclib import jsonclass + +try: + # pylint: disable=F0401,E0611 + # Using cjson + import cjson + _logger.debug("Using cjson as JSON library") + + # Declare cjson methods + def jdumps(obj, encoding='utf-8'): + """ + Serializes ``obj`` to a JSON formatted string, using cjson. + """ + return cjson.encode(obj) + + def jloads(json_string): + """ + Deserializes ``json_string`` (a string containing a JSON document) + to a Python object, using cjson. + """ + return cjson.decode(json_string) + +except ImportError: + # pylint: disable=F0401,E0611 + # Use json or simplejson + try: + import json + _logger.debug("Using json as JSON library") + + except ImportError: + try: + import simplejson as json + _logger.debug("Using simplejson as JSON library") + except ImportError: + _logger.error("No supported JSON library found") + raise ImportError('You must have the cjson, json, or simplejson ' + 'module(s) available.') + + # Declare json methods + if sys.version_info[0] < 3: + def jdumps(obj, encoding='utf-8'): + """ + Serializes ``obj`` to a JSON formatted string. + """ + # Python 2 (explicit encoding) + return json.dumps(obj, encoding=encoding) + + else: + # Python 3 + def jdumps(obj, encoding='utf-8'): + """ + Serializes ``obj`` to a JSON formatted string. + """ + # Python 3 (the encoding parameter has been removed) + return json.dumps(obj) + + def jloads(json_string): + """ + Deserializes ``json_string`` (a string containing a JSON document) + to a Python object. + """ + return json.loads(json_string) + +# ------------------------------------------------------------------------------ +# XMLRPClib re-implementations + + +class ProtocolError(Exception): + """ + JSON-RPC error + + ProtocolError.args[0] can be: + * an error message (string) + * a (code, message) tuple + """ + pass + + +class AppError(ProtocolError): + """ + Application error: the error code is not in the pre-defined ones + + AppError.args[0][0]: Error code + AppError.args[0][1]: Error message or trace + AppError.args[0][2]: Associated data + """ + def data(self): + """ + Retrieves the value found in the 'data' entry of the error, or None + + :return: The data associated to the error, or None + """ + return self.args[0][2] + + +class JSONParser(object): + """ + Default JSON parser + """ + def __init__(self, target): + """ + Associates the target loader to the parser + + :param target: a JSONTarget instance + """ + self.target = target + + def feed(self, data): + """ + Feeds the associated target with the given data + """ + self.target.feed(data) + + def close(self): + """ + Does nothing + """ + pass + + +class JSONTarget(object): + """ + Unmarshalls stream data to a string + """ + def __init__(self): + """ + Sets up the unmarshaller + """ + self.data = [] + + def feed(self, data): + """ + Stores the given raw data into a buffer + """ + # Store raw data as it might not contain whole wide-character + self.data.append(data) + + def close(self): + """ + Unmarshalls the buffered data + """ + if not self.data: + return '' + else: + # Use type to have a valid join (str vs. bytes) + data = type(self.data[0])().join(self.data) + try: + # Convert the whole final string + data = utils.from_bytes(data) + except: + # Try a pass-through + pass + + return data + + +class TransportMixIn(object): + """ Just extends the XMLRPC transport where necessary. """ + # for Python 2.7 support + _connection = None + + # List of non-overridable headers + # Use the configuration to change the content-type + readonly_headers = ('content-length', 'content-type') + + def __init__(self, config=jsonrpclib.config.DEFAULT, context=None): + """ + Sets up the transport + + :param config: A JSONRPClib Config instance + """ + # Store the configuration + self._config = config + + # Store the SSL context + self.context = context + + # Set up the user agent + self.user_agent = config.user_agent + + # Additional headers: list of dictionaries + self.additional_headers = [] + + def push_headers(self, headers): + """ + Adds a dictionary of headers to the additional headers list + + :param headers: A dictionary + """ + self.additional_headers.append(headers) + + def pop_headers(self, headers): + """ + Removes the given dictionary from the additional headers list. + Also validates that given headers are on top of the stack + + :param headers: Headers to remove + :raise AssertionError: The given dictionary is not on the latest stored + in the additional headers list + """ + assert self.additional_headers[-1] == headers + self.additional_headers.pop() + + def emit_additional_headers(self, connection): + """ + Puts headers as is in the request, filtered read only headers + + :param connection: The request connection + """ + additional_headers = {} + + # Prepare the merged dictionary + for headers in self.additional_headers: + additional_headers.update(headers) + + # Remove forbidden keys + for forbidden in self.readonly_headers: + additional_headers.pop(forbidden, None) + + # Reversed order: in the case of multiple headers value definition, + # the latest pushed has priority + for key, value in additional_headers.items(): + key = str(key) + if key.lower() not in self.readonly_headers: + # Only accept replaceable headers + connection.putheader(str(key), str(value)) + + def send_content(self, connection, request_body): + """ + Completes the request headers and sends the request body of a JSON-RPC + request over a HTTPConnection + + :param connection: An HTTPConnection object + :param request_body: JSON-RPC request body + """ + # Convert the body first + request_body = utils.to_bytes(request_body) + + # "static" headers + connection.putheader("Content-Type", self._config.content_type) + connection.putheader("Content-Length", str(len(request_body))) + + # Emit additional headers here in order not to override content-length + self.emit_additional_headers(connection) + + connection.endheaders() + if request_body: + connection.send(request_body) + + def getparser(self): + """ + Create an instance of the parser, and attach it to an unmarshalling + object. Return both objects. + + :return: The parser and unmarshaller instances + """ + target = JSONTarget() + return JSONParser(target), target + + +class Transport(TransportMixIn, XMLTransport): + """ + Mixed-in HTTP transport + """ + pass + + +class SafeTransport(TransportMixIn, XMLSafeTransport): + """ + Mixed-in HTTPS transport + """ + pass + +# ------------------------------------------------------------------------------ + + +class ServerProxy(XMLServerProxy): + """ + Unfortunately, much more of this class has to be copied since + so much of it does the serialization. + """ + def __init__(self, uri, transport=None, encoding=None, + verbose=0, version=None, headers=None, history=None, + config=jsonrpclib.config.DEFAULT, context=None): + """ + Sets up the server proxy + + :param uri: Request URI + :param transport: Custom transport handler + :param encoding: Specified encoding + :param verbose: Log verbosity level + :param version: JSON-RPC specification version + :param headers: Custom additional headers for each request + :param history: History object (for tests) + :param config: A JSONRPClib Config instance + :param context: The optional SSLContext to use + """ + # Store the configuration + self._config = config + self.__version = version or config.version + + schema, uri = splittype(uri) + if schema not in ('http', 'https'): + _logger.error("jsonrpclib only support http(s) URIs, not %s", + schema) + raise IOError('Unsupported JSON-RPC protocol.') + + self.__host, self.__handler = splithost(uri) + if not self.__handler: + # Not sure if this is in the JSON spec? + self.__handler = '/' + + if transport is None: + if schema == 'https': + transport = SafeTransport(config=config, context=context) + else: + transport = Transport(config=config) + self.__transport = transport + + self.__encoding = encoding + self.__verbose = verbose + self.__history = history + + # Global custom headers are injected into Transport + self.__transport.push_headers(headers or {}) + + def _request(self, methodname, params, rpcid=None): + """ + Calls a method on the remote server + + :param methodname: Name of the method to call + :param params: Method parameters + :param rpcid: ID of the remote call + :return: The parsed result of the call + """ + request = dumps(params, methodname, encoding=self.__encoding, + rpcid=rpcid, version=self.__version, + config=self._config) + response = self._run_request(request) + check_for_errors(response) + return response['result'] + + def _request_notify(self, methodname, params, rpcid=None): + """ + Calls a method as a notification + + :param methodname: Name of the method to call + :param params: Method parameters + :param rpcid: ID of the remote call + """ + request = dumps(params, methodname, encoding=self.__encoding, + rpcid=rpcid, version=self.__version, notify=True, + config=self._config) + response = self._run_request(request, notify=True) + check_for_errors(response) + + def _run_request(self, request, notify=False): + """ + Sends the given request to the remote server + + :param request: The request to send + :param notify: Notification request flag (unused) + :return: The response as a parsed JSON object + """ + if self.__history is not None: + self.__history.add_request(request) + + response = self.__transport.request( + self.__host, + self.__handler, + request, + verbose=self.__verbose + ) + + # Here, the XMLRPC library translates a single list + # response to the single value -- should we do the + # same, and require a tuple / list to be passed to + # the response object, or expect the Server to be + # outputting the response appropriately? + + if self.__history is not None: + self.__history.add_response(response) + + if not response: + return None + else: + return_obj = loads(response, self._config) + return return_obj + + def __getattr__(self, name): + """ + Returns a callable object to call the remote service + """ + # Same as original, just with new _Method reference + return _Method(self._request, name) + + def __close(self): + """ + Closes the transport layer + """ + try: + self.__transport.close() + except AttributeError: + # Not available in Python 2.6 + pass + + def __call__(self, attr): + """ + A workaround to get special attributes on the ServerProxy + without interfering with the magic __getattr__ + + (code from xmlrpclib in Python 2.7) + """ + if attr == "close": + return self.__close + + elif attr == "transport": + return self.__transport + + raise AttributeError("Attribute {0} not found".format(attr)) + + @property + def _notify(self): + """ + Like __getattr__, but sending a notification request instead of a call + """ + return _Notify(self._request_notify) + + @contextlib.contextmanager + def _additional_headers(self, headers): + """ + Allows to specify additional headers, to be added inside the with + block. + Example of usage: + + >>> with client._additional_headers({'X-Test' : 'Test'}) as new_client: + ... new_client.method() + ... + >>> # Here old headers are restored + """ + self.__transport.push_headers(headers) + yield self + self.__transport.pop_headers(headers) + +# ------------------------------------------------------------------------------ + + +class _Method(XML_Method): + """ + Some magic to bind an JSON-RPC method to an RPC server. + """ + def __call__(self, *args, **kwargs): + """ + Sends an RPC request and returns the unmarshalled result + """ + if args and kwargs: + raise ProtocolError("Cannot use both positional and keyword " + "arguments (according to JSON-RPC spec.)") + if args: + return self.__send(self.__name, args) + else: + return self.__send(self.__name, kwargs) + + def __getattr__(self, name): + """ + Returns a Method object for nested calls + """ + if name == "__name__": + return self.__name + return _Method(self.__send, "{0}.{1}".format(self.__name, name)) + + +class _Notify(object): + """ + Same as _Method, but to send notifications + """ + def __init__(self, request): + """ + Sets the method to call to send a request to the server + """ + self._request = request + + def __getattr__(self, name): + """ + Returns a Method object, to be called as a notification + """ + return _Method(self._request, name) + +# ------------------------------------------------------------------------------ +# Batch implementation + + +class MultiCallMethod(object): + """ + Stores calls made to a MultiCall object for batch execution + """ + def __init__(self, method, notify=False, config=jsonrpclib.config.DEFAULT): + """ + Sets up the store + + :param method: Name of the method to call + :param notify: Notification flag + :param config: Request configuration + """ + self.method = method + self.params = [] + self.notify = notify + self._config = config + + def __call__(self, *args, **kwargs): + """ + Normalizes call parameters + """ + if kwargs and args: + raise ProtocolError('JSON-RPC does not support both ' + + 'positional and keyword arguments.') + if kwargs: + self.params = kwargs + else: + self.params = args + + def request(self, encoding=None, rpcid=None): + """ + Returns the request object as JSON-formatted string + """ + return dumps(self.params, self.method, version=2.0, + encoding=encoding, rpcid=rpcid, notify=self.notify, + config=self._config) + + def __repr__(self): + """ + String representation + """ + return str(self.request()) + + def __getattr__(self, method): + """ + Updates the object for a nested call + """ + self.method = "{0}.{1}".format(self.method, method) + return self + + +class MultiCallNotify(object): + """ + Same as MultiCallMethod but for notifications + """ + def __init__(self, multicall, config=jsonrpclib.config.DEFAULT): + """ + Sets ip the store + + :param multicall: The parent MultiCall instance + :param config: Request configuration + """ + self.multicall = multicall + self._config = config + + def __getattr__(self, name): + """ + Returns the MultiCallMethod to use as a notification + """ + new_job = MultiCallMethod(name, notify=True, config=self._config) + self.multicall._job_list.append(new_job) + return new_job + + +class MultiCallIterator(object): + """ + Iterates over the results of a MultiCall. + Exceptions are raised in response to JSON-RPC faults + """ + def __init__(self, results): + """ + Sets up the results store + """ + self.results = results + + def __get_result(self, item): + """ + Checks for error and returns the "real" result stored in a MultiCall + result. + """ + check_for_errors(item) + return item['result'] + + def __iter__(self): + """ + Iterates over all results + """ + for item in self.results: + yield self.__get_result(item) + raise StopIteration + + def __getitem__(self, i): + """ + Returns the i-th object of the results + """ + return self.__get_result(self.results[i]) + + def __len__(self): + """ + Returns the number of results stored + """ + return len(self.results) + + +class MultiCall(object): + """ + server -> a object used to boxcar method calls, where server should be a + ServerProxy object. + + Methods can be added to the MultiCall using normal + method call syntax e.g.: + + multicall = MultiCall(server_proxy) + multicall.add(2,3) + multicall.get_address("Guido") + + To execute the multicall, call the MultiCall object e.g.: + + add_result, address = multicall() + """ + def __init__(self, server, config=jsonrpclib.config.DEFAULT): + """ + Sets up the multicall + + :param server: A ServerProxy object + :param config: Request configuration + """ + self._server = server + self._job_list = [] + self._config = config + + def _request(self): + """ + Sends the request to the server and returns the responses + + :return: A MultiCallIterator object + """ + if len(self._job_list) < 1: + # Should we alert? This /is/ pretty obvious. + return + request_body = "[ {0} ]".format( + ','.join(job.request() for job in self._job_list)) + responses = self._server._run_request(request_body) + del self._job_list[:] + if not responses: + responses = [] + return MultiCallIterator(responses) + + @property + def _notify(self): + """ + Prepares a notification call + """ + return MultiCallNotify(self, self._config) + + def __getattr__(self, name): + """ + Registers a method call + """ + new_job = MultiCallMethod(name, config=self._config) + self._job_list.append(new_job) + return new_job + + __call__ = _request + +# These lines conform to xmlrpclib's "compatibility" line. +# Not really sure if we should include these, but oh well. +Server = ServerProxy + +# ------------------------------------------------------------------------------ + + +class Fault(object): + """ + JSON-RPC error class + """ + def __init__(self, code=-32000, message='Server error', rpcid=None, + config=jsonrpclib.config.DEFAULT, data=None): + """ + Sets up the error description + + :param code: Fault code + :param message: Associated message + :param rpcid: Request ID + :param config: A JSONRPClib Config instance + :param data: Extra information added to an error description + """ + self.faultCode = code + self.faultString = message + self.rpcid = rpcid + self.config = config + self.data = data + + def error(self): + """ + Returns the error as a dictionary + + :returns: A {'code', 'message'} dictionary + """ + return {'code': self.faultCode, 'message': self.faultString, + 'data': self.data} + + def response(self, rpcid=None, version=None): + """ + Returns the error as a JSON-RPC response string + + :param rpcid: Forced request ID + :param version: JSON-RPC version + :return: A JSON-RPC response string + """ + if not version: + version = self.config.version + + if rpcid: + self.rpcid = rpcid + + return dumps(self, methodresponse=True, rpcid=self.rpcid, + version=version, config=self.config) + + def dump(self, rpcid=None, version=None): + """ + Returns the error as a JSON-RPC response dictionary + + :param rpcid: Forced request ID + :param version: JSON-RPC version + :return: A JSON-RPC response dictionary + """ + if not version: + version = self.config.version + + if rpcid: + self.rpcid = rpcid + + return dump(self, is_response=True, rpcid=self.rpcid, + version=version, config=self.config) + + def __repr__(self): + """ + String representation + """ + return '<Fault {0}: {1}>'.format(self.faultCode, self.faultString) + + +class Payload(object): + """ + JSON-RPC content handler + """ + def __init__(self, rpcid=None, version=None, + config=jsonrpclib.config.DEFAULT): + """ + Sets up the JSON-RPC handler + + :param rpcid: Request ID + :param version: JSON-RPC version + :param config: A JSONRPClib Config instance + """ + if not version: + version = config.version + + self.id = rpcid + self.version = float(version) + + def request(self, method, params=None): + """ + Prepares a method call request + + :param method: Method name + :param params: Method parameters + :return: A JSON-RPC request dictionary + """ + if not isinstance(method, utils.string_types): + raise ValueError('Method name must be a string.') + + if not self.id: + # Generate a request ID + self.id = str(uuid.uuid4()) + + request = {'id': self.id, 'method': method} + if params or self.version < 1.1: + request['params'] = params or [] + + if self.version >= 2: + request['jsonrpc'] = str(self.version) + + return request + + def notify(self, method, params=None): + """ + Prepares a notification request + + :param method: Notification name + :param params: Notification parameters + :return: A JSON-RPC notification dictionary + """ + # Prepare the request dictionary + request = self.request(method, params) + + # Remove the request ID, as it's a notification + if self.version >= 2: + del request['id'] + else: + request['id'] = None + + return request + + def response(self, result=None): + """ + Prepares a response dictionary + + :param result: The result of method call + :return: A JSON-RPC response dictionary + """ + response = {'result': result, 'id': self.id} + + if self.version >= 2: + response['jsonrpc'] = str(self.version) + else: + response['error'] = None + + return response + + def error(self, code=-32000, message='Server error.', data=None): + """ + Prepares an error dictionary + + :param code: Error code + :param message: Error message + :return: A JSON-RPC error dictionary + """ + error = self.response() + if self.version >= 2: + del error['result'] + else: + error['result'] = None + error['error'] = {'code': code, 'message': message} + if data is not None: + error['error']['data'] = data + return error + +# ------------------------------------------------------------------------------ + + +def dump(params=None, methodname=None, rpcid=None, version=None, + is_response=None, is_notify=None, config=jsonrpclib.config.DEFAULT): + """ + Prepares a JSON-RPC dictionary (request, notification, response or error) + + :param params: Method parameters (if a method name is given) or a Fault + :param methodname: Method name + :param rpcid: Request ID + :param version: JSON-RPC version + :param is_response: If True, this is a response dictionary + :param is_notify: If True, this is a notification request + :param config: A JSONRPClib Config instance + :return: A JSON-RPC dictionary + """ + # Default version + if not version: + version = config.version + + if not is_response and params is None: + params = [] + + # Validate method name and parameters + valid_params = [utils.TupleType, utils.ListType, utils.DictType, Fault] + if is_response: + valid_params.append(type(None)) + + if isinstance(methodname, utils.string_types) and \ + not isinstance(params, tuple(valid_params)): + """ + If a method, and params are not in a listish or a Fault, + error out. + """ + raise TypeError("Params must be a dict, list, tuple " + "or Fault instance.") + + # Prepares the JSON-RPC content + payload = Payload(rpcid=rpcid, version=version) + + if isinstance(params, Fault): + # Prepare an error dictionary + # pylint: disable=E1103 + return payload.error(params.faultCode, params.faultString, params.data) + + if not isinstance(methodname, utils.string_types) and not is_response: + # Neither a request nor a response + raise ValueError('Method name must be a string, or is_response ' + 'must be set to True.') + + if config.use_jsonclass: + # Use jsonclass to convert the parameters + params = jsonclass.dump(params, config=config) + + if is_response: + # Prepare a response dictionary + if rpcid is None: + # A response must have a request ID + raise ValueError('A method response must have an rpcid.') + return payload.response(params) + + if is_notify: + # Prepare a notification dictionary + return payload.notify(methodname, params) + else: + # Prepare a method call dictionary + return payload.request(methodname, params) + + +def dumps(params=None, methodname=None, methodresponse=None, + encoding=None, rpcid=None, version=None, notify=None, + config=jsonrpclib.config.DEFAULT): + """ + Prepares a JSON-RPC request/response string + + :param params: Method parameters (if a method name is given) or a Fault + :param methodname: Method name + :param methodresponse: If True, this is a response dictionary + :param encoding: Result string encoding + :param rpcid: Request ID + :param version: JSON-RPC version + :param notify: If True, this is a notification request + :param config: A JSONRPClib Config instance + :return: A JSON-RPC dictionary + """ + # Prepare the dictionary + request = dump(params, methodname, rpcid, version, methodresponse, notify, + config) + + # Returns it as a JSON string + return jdumps(request, encoding=encoding or "UTF-8") + + +def load(data, config=jsonrpclib.config.DEFAULT): + """ + Loads a JSON-RPC request/response dictionary. Calls jsonclass to load beans + + :param data: A JSON-RPC dictionary + :param config: A JSONRPClib Config instance (or None for default values) + :return: A parsed dictionary or None + """ + if data is None: + # Notification + return None + + # if the above raises an error, the implementing server code + # should return something like the following: + # { 'jsonrpc':'2.0', 'error': fault.error(), id: None } + if config.use_jsonclass: + # Convert beans + data = jsonclass.load(data, config.classes) + + return data + + +def loads(data, config=jsonrpclib.config.DEFAULT): + """ + Loads a JSON-RPC request/response string. Calls jsonclass to load beans + + :param data: A JSON-RPC string + :param config: A JSONRPClib Config instance (or None for default values) + :return: A parsed dictionary or None + """ + if data == '': + # Notification + return None + + # Parse the JSON dictionary + result = jloads(data) + + # Load the beans + return load(result, config) + +# ------------------------------------------------------------------------------ + + +def check_for_errors(result): + """ + Checks if a result dictionary signals an error + + :param result: A result dictionary + :raise TypeError: Invalid parameter + :raise NotImplementedError: Unknown JSON-RPC version + :raise ValueError: Invalid dictionary content + :raise ProtocolError: An error occurred on the server side + :return: The result parameter + """ + if not result: + # Notification + return result + + if not isinstance(result, utils.DictType): + # Invalid argument + raise TypeError('Response is not a dict.') + + if 'jsonrpc' in result and float(result['jsonrpc']) > 2.0: + # Unknown JSON-RPC version + raise NotImplementedError('JSON-RPC version not yet supported.') + + if 'result' not in result and 'error' not in result: + # Invalid dictionary content + raise ValueError('Response does not have a result or error key.') + + if 'error' in result and result['error']: + # Server-side error + if 'code' in result['error']: + # Code + Message + code = result['error']['code'] + try: + # Get the message (jsonrpclib) + message = result['error']['message'] + except KeyError: + # Get the trace (jabsorb) + message = result['error'].get('trace', '<no error message>') + + if -32700 <= code <= -32000: + # Pre-defined errors + # See http://www.jsonrpc.org/specification#error_object + raise ProtocolError((code, message)) + else: + # Application error + data = result['error'].get('data', None) + raise AppError((code, message, data)) + + elif isinstance(result['error'], dict) and len(result['error']) == 1: + # Error with a single entry ('reason', ...): use its content + error_key = result['error'].keys()[0] + raise ProtocolError(result['error'][error_key]) + + else: + # Use the raw error content + raise ProtocolError(result['error']) + + return result + + +def isbatch(request): + """ + Tests if the given request is a batch call, i.e. a list of multiple calls + :param request: a JSON-RPC request object + :return: True if the request is a batch call + """ + if not isinstance(request, (utils.ListType, utils.TupleType)): + # Not a list: not a batch call + return False + elif len(request) < 1: + # Only one request: not a batch call + return False + elif not isinstance(request[0], utils.DictType): + # One of the requests is not a dictionary, i.e. a JSON Object + # therefore it is not a valid JSON-RPC request + return False + elif 'jsonrpc' not in request[0].keys(): + # No "jsonrpc" version in the JSON object: not a request + return False + + try: + version = float(request[0]['jsonrpc']) + except ValueError: + # Bad version of JSON-RPC + raise ProtocolError('"jsonrpc" key must be a float(able) value.') + + if version < 2: + # Batch call were not supported before JSON-RPC 2.0 + return False + + return True + + +def isnotification(request): + """ + Tests if the given request is a notification + + :param request: A request dictionary + :return: True if the request is a notification + """ + if 'id' not in request: + # 2.0 notification + return True + + if request['id'] is None: + # 1.0 notification + return True + + return False diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/threadpool.py b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/threadpool.py new file mode 100644 index 00000000..a38b5b83 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/threadpool.py @@ -0,0 +1,490 @@ +#!/usr/bin/env python +# -- Content-Encoding: UTF-8 -- +""" +Cached thread pool, inspired from Pelix/iPOPO Thread Pool + +:author: Thomas Calmant +:copyright: Copyright 2015, isandlaTech +:license: Apache License 2.0 +:version: 0.2.5 + +.. + + Copyright 2015 isandlaTech + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +# Documentation strings format +__docformat__ = "restructuredtext en" + +# Module version +__version_info__ = (0, 2, 5) +__version__ = ".".join(str(x) for x in __version_info__) + +# ------------------------------------------------------------------------------ + +# Standard library +import logging +import threading + +try: + # Python 3 + # pylint: disable=F0401 + import queue +except ImportError: + # Python 2 + # pylint: disable=F0401 + import Queue as queue + +# ------------------------------------------------------------------------------ + + +class EventData(object): + """ + A threading event with some associated data + """ + def __init__(self): + """ + Sets up the event + """ + self.__event = threading.Event() + self.__data = None + self.__exception = None + + @property + def data(self): + """ + Returns the associated value + """ + return self.__data + + @property + def exception(self): + """ + Returns the exception used to stop the wait() method + """ + return self.__exception + + def clear(self): + """ + Clears the event + """ + self.__event.clear() + self.__data = None + self.__exception = None + + def is_set(self): + """ + Checks if the event is set + """ + return self.__event.is_set() + + def set(self, data=None): + """ + Sets the event + """ + self.__data = data + self.__exception = None + self.__event.set() + + def raise_exception(self, exception): + """ + Raises an exception in wait() + + :param exception: An Exception object + """ + self.__data = None + self.__exception = exception + self.__event.set() + + def wait(self, timeout=None): + """ + Waits for the event or for the timeout + + :param timeout: Wait timeout (in seconds) + :return: True if the event as been set, else False + """ + # The 'or' part is for Python 2.6 + result = self.__event.wait(timeout) or self.__event.is_set() + # pylint: disable=E0702 + # Pylint seems to miss the "is None" check below + if self.__exception is None: + return result + else: + raise self.__exception + + +class FutureResult(object): + """ + An object to wait for the result of a threaded execution + """ + def __init__(self, logger=None): + """ + Sets up the FutureResult object + + :param logger: The Logger to use in case of error (optional) + """ + self._logger = logger or logging.getLogger(__name__) + self._done_event = EventData() + self.__callback = None + self.__extra = None + + def __notify(self): + """ + Notify the given callback about the result of the execution + """ + if self.__callback is not None: + try: + self.__callback(self._done_event.data, + self._done_event.exception, + self.__extra) + except Exception as ex: + self._logger.exception("Error calling back method: %s", ex) + + def set_callback(self, method, extra=None): + """ + Sets a callback method, called once the result has been computed or in + case of exception. + + The callback method must have the following signature: + ``callback(result, exception, extra)``. + + :param method: The method to call back in the end of the execution + :param extra: Extra parameter to be given to the callback method + """ + self.__callback = method + self.__extra = extra + if self._done_event.is_set(): + # The execution has already finished + self.__notify() + + def execute(self, method, args, kwargs): + """ + Execute the given method and stores its result. + The result is considered "done" even if the method raises an exception + + :param method: The method to execute + :param args: Method positional arguments + :param kwargs: Method keyword arguments + :raise Exception: The exception raised by the method + """ + # Normalize arguments + if args is None: + args = [] + + if kwargs is None: + kwargs = {} + + try: + # Call the method + result = method(*args, **kwargs) + except Exception as ex: + # Something went wrong: propagate to the event and to the caller + self._done_event.raise_exception(ex) + raise + else: + # Store the result + self._done_event.set(result) + finally: + # In any case: notify the call back (if any) + self.__notify() + + def done(self): + """ + Returns True if the job has finished, else False + """ + return self._done_event.is_set() + + def result(self, timeout=None): + """ + Waits up to timeout for the result the threaded job. + Returns immediately the result if the job has already been done. + + :param timeout: The maximum time to wait for a result (in seconds) + :raise OSError: The timeout raised before the job finished + :raise Exception: The exception encountered during the call, if any + """ + if self._done_event.wait(timeout): + return self._done_event.data + else: + raise OSError("Timeout raised") + +# ------------------------------------------------------------------------------ + + +class ThreadPool(object): + """ + Executes the tasks stored in a FIFO in a thread pool + """ + def __init__(self, max_threads, min_threads=1, queue_size=0, timeout=60, + logname=None): + """ + Sets up the thread pool. + + Threads are kept alive 60 seconds (timeout argument). + + :param max_threads: Maximum size of the thread pool + :param min_threads: Minimum size of the thread pool + :param queue_size: Size of the task queue (0 for infinite) + :param timeout: Queue timeout (in seconds, 60s by default) + :param logname: Name of the logger + :raise ValueError: Invalid number of threads + """ + # Validate parameters + try: + max_threads = int(max_threads) + if max_threads < 1: + raise ValueError("Pool size must be greater than 0") + except (TypeError, ValueError) as ex: + raise ValueError("Invalid pool size: {0}".format(ex)) + + try: + min_threads = int(min_threads) + if min_threads < 0: + min_threads = 0 + elif min_threads > max_threads: + min_threads = max_threads + except (TypeError, ValueError) as ex: + raise ValueError("Invalid pool size: {0}".format(ex)) + + # The logger + self._logger = logging.getLogger(logname or __name__) + + # The loop control event + self._done_event = threading.Event() + self._done_event.set() + + # The task queue + try: + queue_size = int(queue_size) + except (TypeError, ValueError): + # Not a valid integer + queue_size = 0 + + self._queue = queue.Queue(queue_size) + self._timeout = timeout + self.__lock = threading.RLock() + + # The thread pool + self._min_threads = min_threads + self._max_threads = max_threads + self._threads = [] + + # Thread count + self._thread_id = 0 + + # Current number of threads, active and alive + self.__nb_threads = 0 + self.__nb_active_threads = 0 + + def start(self): + """ + Starts the thread pool. Does nothing if the pool is already started. + """ + if not self._done_event.is_set(): + # Stop event not set: we're running + return + + # Clear the stop event + self._done_event.clear() + + # Compute the number of threads to start to handle pending tasks + nb_pending_tasks = self._queue.qsize() + if nb_pending_tasks > self._max_threads: + nb_threads = self._max_threads + elif nb_pending_tasks < self._min_threads: + nb_threads = self._min_threads + else: + nb_threads = nb_pending_tasks + + # Create the threads + for _ in range(nb_threads): + self.__start_thread() + + def __start_thread(self): + """ + Starts a new thread, if possible + """ + with self.__lock: + if self.__nb_threads >= self._max_threads: + # Can't create more threads + return False + + if self._done_event.is_set(): + # We're stopped: do nothing + return False + + # Prepare thread and start it + name = "{0}-{1}".format(self._logger.name, self._thread_id) + self._thread_id += 1 + + thread = threading.Thread(target=self.__run, name=name) + thread.daemon = True + self._threads.append(thread) + thread.start() + return True + + def stop(self): + """ + Stops the thread pool. Does nothing if the pool is already stopped. + """ + if self._done_event.is_set(): + # Stop event set: we're stopped + return + + # Set the stop event + self._done_event.set() + + with self.__lock: + # Add something in the queue (to unlock the join()) + try: + for _ in self._threads: + self._queue.put(self._done_event, True, self._timeout) + except queue.Full: + # There is already something in the queue + pass + + # Copy the list of threads to wait for + threads = self._threads[:] + + # Join threads outside the lock + for thread in threads: + while thread.is_alive(): + # Wait 3 seconds + thread.join(3) + if thread.is_alive(): + # Thread is still alive: something might be wrong + self._logger.warning("Thread %s is still alive...", + thread.name) + + # Clear storage + del self._threads[:] + self.clear() + + def enqueue(self, method, *args, **kwargs): + """ + Queues a task in the pool + + :param method: Method to call + :return: A FutureResult object, to get the result of the task + :raise ValueError: Invalid method + :raise Full: The task queue is full + """ + if not hasattr(method, '__call__'): + raise ValueError("{0} has no __call__ member." + .format(method.__name__)) + + # Prepare the future result object + future = FutureResult(self._logger) + + # Use a lock, as we might be "resetting" the queue + with self.__lock: + # Add the task to the queue + self._queue.put((method, args, kwargs, future), True, + self._timeout) + + if self.__nb_active_threads == self.__nb_threads: + # All threads are taken: start a new one + self.__start_thread() + + return future + + def clear(self): + """ + Empties the current queue content. + Returns once the queue have been emptied. + """ + with self.__lock: + # Empty the current queue + try: + while True: + self._queue.get_nowait() + self._queue.task_done() + except queue.Empty: + # Queue is now empty + pass + + # Wait for the tasks currently executed + self.join() + + def join(self, timeout=None): + """ + Waits for all the tasks to be executed + + :param timeout: Maximum time to wait (in seconds) + :return: True if the queue has been emptied, else False + """ + if self._queue.empty(): + # Nothing to wait for... + return True + elif timeout is None: + # Use the original join + self._queue.join() + return True + else: + # Wait for the condition + with self._queue.all_tasks_done: + self._queue.all_tasks_done.wait(timeout) + return not bool(self._queue.unfinished_tasks) + + def __run(self): + """ + The main loop + """ + with self.__lock: + self.__nb_threads += 1 + + while not self._done_event.is_set(): + try: + # Wait for an action (blocking) + task = self._queue.get(True, self._timeout) + if task is self._done_event: + # Stop event in the queue: get out + self._queue.task_done() + with self.__lock: + self.__nb_threads -= 1 + return + except queue.Empty: + # Nothing to do yet + pass + else: + with self.__lock: + self.__nb_active_threads += 1 + + # Extract elements + method, args, kwargs, future = task + try: + # Call the method + future.execute(method, args, kwargs) + except Exception as ex: + self._logger.exception("Error executing %s: %s", + method.__name__, ex) + finally: + # Mark the action as executed + self._queue.task_done() + + # Thread is not active anymore + self.__nb_active_threads -= 1 + + # Clean up thread if necessary + with self.__lock: + if self.__nb_threads > self._min_threads: + # No more work for this thread, and we're above the + # minimum number of threads: stop this one + self.__nb_threads -= 1 + return + + with self.__lock: + # Thread stops + self.__nb_threads -= 1 diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/utils.py b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/utils.py new file mode 100644 index 00000000..31183742 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib/utils.py @@ -0,0 +1,122 @@ +#!/usr/bin/python +# -- Content-Encoding: UTF-8 -- +""" +Utility methods, for compatibility between Python version + +:author: Thomas Calmant +:copyright: Copyright 2015, isandlaTech +:license: Apache License 2.0 +:version: 0.2.5 + +.. + + Copyright 2015 isandlaTech + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +# Module version +__version_info__ = (0, 2, 5) +__version__ = ".".join(str(x) for x in __version_info__) + +# Documentation strings format +__docformat__ = "restructuredtext en" + +# ------------------------------------------------------------------------------ + +import sys + +# ------------------------------------------------------------------------------ + +if sys.version_info[0] < 3: + # Python 2 + import types + try: + string_types = ( + types.StringType, + types.UnicodeType + ) + except NameError: + # Python built without unicode support + string_types = (types.StringType,) + + numeric_types = ( + types.IntType, + types.LongType, + types.FloatType + ) + + def to_bytes(string): + """ + Converts the given string into bytes + """ + if type(string) is unicode: + return str(string) + return string + + def from_bytes(data): + """ + Converts the given bytes into a string + """ + if type(data) is str: + return data + return str(data) + +else: + # Python 3 + string_types = ( + bytes, + str + ) + + numeric_types = ( + int, + float + ) + + def to_bytes(string): + """ + Converts the given string into bytes + """ + if type(string) is bytes: + return string + return bytes(string, "UTF-8") + + def from_bytes(data): + """ + Converts the given bytes into a string + """ + if type(data) is str: + return data + return str(data, "UTF-8") + +# ------------------------------------------------------------------------------ +# Common + +DictType = dict + +ListType = list +TupleType = tuple + +iterable_types = ( + list, + set, frozenset, + tuple +) + +value_types = ( + bool, + type(None) +) + +primitive_types = string_types + numeric_types + value_types diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/PKG-INFO b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/PKG-INFO new file mode 100644 index 00000000..5dce6b1c --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/PKG-INFO @@ -0,0 +1,460 @@ +Metadata-Version: 1.1 +Name: jsonrpclib-pelix +Version: 0.2.5 +Summary: This project is an implementation of the JSON-RPC v2.0 specification (backwards-compatible) as a client library, for Python 2.6+ and Python 3.This version is a fork of jsonrpclib by Josh Marshall, usable with Pelix remote services. +Home-page: http://github.com/tcalmant/jsonrpclib/ +Author: Thomas Calmant +Author-email: thomas.calmant+github@gmail.com +License: Apache License 2.0 +Description: JSONRPClib (patched for Pelix) + ############################## + + .. image:: https://pypip.in/license/jsonrpclib-pelix/badge.svg + :target: https://pypi.python.org/pypi/jsonrpclib-pelix/ + + .. image:: https://travis-ci.org/tcalmant/jsonrpclib.svg?branch=master + :target: https://travis-ci.org/tcalmant/jsonrpclib + + .. image:: https://coveralls.io/repos/tcalmant/jsonrpclib/badge.svg?branch=master + :target: https://coveralls.io/r/tcalmant/jsonrpclib?branch=master + + + This library is an implementation of the JSON-RPC specification. + It supports both the original 1.0 specification, as well as the + new (proposed) 2.0 specification, which includes batch submission, keyword + arguments, etc. + + It is licensed under the Apache License, Version 2.0 + (http://www.apache.org/licenses/LICENSE-2.0.html). + + + About this version + ****************** + + This is a patched version of the original ``jsonrpclib`` project by + Josh Marshall, available at https://github.com/joshmarshall/jsonrpclib. + + The suffix *-pelix* only indicates that this version works with Pelix Remote + Services, but it is **not** a Pelix specific implementation. + + * This version adds support for Python 3, staying compatible with Python 2. + * It is now possible to use the dispatch_method argument while extending + the SimpleJSONRPCDispatcher, to use a custom dispatcher. + This allows to use this package by Pelix Remote Services. + * It can use thread pools to control the number of threads spawned to handle + notification requests and clients connections. + * The modifications added in other forks of this project have been added: + + * From https://github.com/drdaeman/jsonrpclib: + + * Improved JSON-RPC 1.0 support + * Less strict error response handling + + * From https://github.com/tuomassalo/jsonrpclib: + + * In case of a non-pre-defined error, raise an AppError and give access to + *error.data* + + * From https://github.com/dejw/jsonrpclib: + + * Custom headers can be sent with request and associated tests + + * The support for Unix sockets has been removed, as it is not trivial to convert + to Python 3 (and I don't use them) + * This version cannot be installed with the original ``jsonrpclib``, as it uses + the same package name. + + + Summary + ******* + + This library implements the JSON-RPC 2.0 proposed specification in pure Python. + It is designed to be as compatible with the syntax of ``xmlrpclib`` as possible + (it extends where possible), so that projects using ``xmlrpclib`` could easily + be modified to use JSON and experiment with the differences. + + It is backwards-compatible with the 1.0 specification, and supports all of the + new proposed features of 2.0, including: + + * Batch submission (via MultiCall) + * Keyword arguments + * Notifications (both in a batch and 'normal') + * Class translation using the ``__jsonclass__`` key. + + I've added a "SimpleJSONRPCServer", which is intended to emulate the + "SimpleXMLRPCServer" from the default Python distribution. + + + Requirements + ************ + + It supports ``cjson`` and ``simplejson``, and looks for the parsers in that + order (searching first for ``cjson``, then for the *built-in* ``json`` in 2.6+, + and then the ``simplejson`` external library). + One of these must be installed to use this library, although if you have a + standard distribution of 2.6+, you should already have one. + Keep in mind that ``cjson`` is supposed to be the quickest, I believe, so if + you are going for full-on optimization you may want to pick it up. + + Since library uses ``contextlib`` module, you should have at least Python 2.5 + installed. + + + Installation + ************ + + You can install this from PyPI with one of the following commands (sudo + may be required): + + .. code-block:: console + + easy_install jsonrpclib-pelix + pip install jsonrpclib-pelix + + Alternatively, you can download the source from the GitHub repository + at http://github.com/tcalmant/jsonrpclib and manually install it + with the following commands: + + .. code-block:: console + + git clone git://github.com/tcalmant/jsonrpclib.git + cd jsonrpclib + python setup.py install + + + SimpleJSONRPCServer + ******************* + + This is identical in usage (or should be) to the SimpleXMLRPCServer in the + Python standard library. Some of the differences in features are that it + obviously supports notification, batch calls, class translation (if left on), + etc. + Note: The import line is slightly different from the regular SimpleXMLRPCServer, + since the SimpleJSONRPCServer is distributed within the ``jsonrpclib`` library. + + .. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + + server = SimpleJSONRPCServer(('localhost', 8080)) + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_function(lambda x: x, 'ping') + server.serve_forever() + + To start protect the server with SSL, use the following snippet: + + .. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + + # Setup the SSL socket + server = SimpleJSONRPCServer(('localhost', 8080), bind_and_activate=False) + server.socket = ssl.wrap_socket(server.socket, certfile='server.pem', + server_side=True) + server.server_bind() + server.server_activate() + + # ... register functions + # Start the server + server.serve_forever() + + + Notification Thread Pool + ======================== + + By default, notification calls are handled in the request handling thread. + It is possible to use a thread pool to handle them, by giving it to the server + using the ``set_notification_pool()`` method: + + .. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import SimpleJSONRPCServer + from jsonrpclib.threadpool import ThreadPool + + # Setup the thread pool: between 0 and 10 threads + pool = ThreadPool(max_threads=10, min_threads=0) + + # Don't forget to start it + pool.start() + + # Setup the server + server = SimpleJSONRPCServer(('localhost', 8080), config) + server.set_notification_pool(pool) + + # Register methods + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_function(lambda x: x, 'ping') + + try: + server.serve_forever() + finally: + # Stop the thread pool (let threads finish their current task) + pool.stop() + server.set_notification_pool(None) + + + Threaded server + =============== + + It is also possible to use a thread pool to handle clients requests, using the + ``PooledJSONRPCServer`` class. + By default, this class uses pool of 0 to 30 threads. A custom pool can be given + with the ``thread_pool`` parameter of the class constructor. + + The notification pool and the request pool are different: by default, a server + with a request pool doesn't have a notification pool. + + .. code-block:: python + + from jsonrpclib.SimpleJSONRPCServer import PooledJSONRPCServer + from jsonrpclib.threadpool import ThreadPool + + # Setup the notification and request pools + nofif_pool = ThreadPool(max_threads=10, min_threads=0) + request_pool = ThreadPool(max_threads=50, min_threads=10) + + # Don't forget to start them + nofif_pool.start() + request_pool.start() + + # Setup the server + server = PooledJSONRPCServer(('localhost', 8080), config, + thread_pool=request_pool) + server.set_notification_pool(nofif_pool) + + # Register methods + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_function(lambda x: x, 'ping') + + try: + server.serve_forever() + finally: + # Stop the thread pools (let threads finish their current task) + request_pool.stop() + nofif_pool.stop() + server.set_notification_pool(None) + + Client Usage + ************ + + This is (obviously) taken from a console session. + + .. code-block:: python + + >>> import jsonrpclib + >>> server = jsonrpclib.ServerProxy('http://localhost:8080') + >>> server.add(5,6) + 11 + >>> server.add(x=5, y=10) + 15 + >>> server._notify.add(5,6) + # No result returned... + >>> batch = jsonrpclib.MultiCall(server) + >>> batch.add(5, 6) + >>> batch.ping({'key':'value'}) + >>> batch._notify.add(4, 30) + >>> results = batch() + >>> for result in results: + >>> ... print(result) + 11 + {'key': 'value'} + # Note that there are only two responses -- this is according to spec. + + # Clean up + >>> server('close')() + + # Using client history + >>> history = jsonrpclib.history.History() + >>> server = jsonrpclib.ServerProxy('http://localhost:8080', history=history) + >>> server.add(5,6) + 11 + >>> print(history.request) + {"id": "f682b956-c8e1-4506-9db4-29fe8bc9fcaa", "jsonrpc": "2.0", + "method": "add", "params": [5, 6]} + >>> print(history.response) + {"id": "f682b956-c8e1-4506-9db4-29fe8bc9fcaa", "jsonrpc": "2.0", + "result": 11} + + # Clean up + >>> server('close')() + + If you need 1.0 functionality, there are a bunch of places you can pass that in, + although the best is just to give a specific configuration to + ``jsonrpclib.ServerProxy``: + + .. code-block:: python + + >>> import jsonrpclib + >>> jsonrpclib.config.DEFAULT.version + 2.0 + >>> config = jsonrpclib.config.Config(version=1.0) + >>> history = jsonrpclib.history.History() + >>> server = jsonrpclib.ServerProxy('http://localhost:8080', config=config, + history=history) + >>> server.add(7, 10) + 17 + >>> print(history.request) + {"id": "827b2923-5b37-49a5-8b36-e73920a16d32", + "method": "add", "params": [7, 10]} + >>> print(history.response) + {"id": "827b2923-5b37-49a5-8b36-e73920a16d32", "error": null, "result": 17} + >>> server('close')() + + The equivalent ``loads`` and ``dumps`` functions also exist, although with minor + modifications. The ``dumps`` arguments are almost identical, but it adds three + arguments: ``rpcid`` for the 'id' key, ``version`` to specify the JSON-RPC + compatibility, and ``notify`` if it's a request that you want to be a + notification. + + Additionally, the ``loads`` method does not return the params and method like + ``xmlrpclib``, but instead a.) parses for errors, raising ProtocolErrors, and + b.) returns the entire structure of the request / response for manual parsing. + + + Additional headers + ****************** + + If your remote service requires custom headers in request, you can pass them + as as a ``headers`` keyword argument, when creating the ``ServerProxy``: + + .. code-block:: python + + >>> import jsonrpclib + >>> server = jsonrpclib.ServerProxy("http://localhost:8080", + headers={'X-Test' : 'Test'}) + + You can also put additional request headers only for certain method invocation: + + .. code-block:: python + + >>> import jsonrpclib + >>> server = jsonrpclib.Server("http://localhost:8080") + >>> with server._additional_headers({'X-Test' : 'Test'}) as test_server: + ... test_server.ping(42) + ... + >>> # X-Test header will be no longer sent in requests + + Of course ``_additional_headers`` contexts can be nested as well. + + + Class Translation + ***************** + + I've recently added "automatic" class translation support, although it is + turned off by default. This can be devastatingly slow if improperly used, so + the following is just a short list of things to keep in mind when using it. + + * Keep It (the object) Simple Stupid. (for exceptions, keep reading.) + * Do not require init params (for exceptions, keep reading) + * Getter properties without setters could be dangerous (read: not tested) + + If any of the above are issues, use the _serialize method. (see usage below) + The server and client must BOTH have use_jsonclass configuration item on and + they must both have access to the same libraries used by the objects for + this to work. + + If you have excessively nested arguments, it would be better to turn off the + translation and manually invoke it on specific objects using + ``jsonrpclib.jsonclass.dump`` / ``jsonrpclib.jsonclass.load`` (since the default + behavior recursively goes through attributes and lists / dicts / tuples). + + Sample file: *test_obj.py* + + .. code-block:: python + + # This object is /very/ simple, and the system will look through the + # attributes and serialize what it can. + class TestObj(object): + foo = 'bar' + + # This object requires __init__ params, so it uses the _serialize method + # and returns a tuple of init params and attribute values (the init params + # can be a dict or a list, but the attribute values must be a dict.) + class TestSerial(object): + foo = 'bar' + def __init__(self, *args): + self.args = args + def _serialize(self): + return (self.args, {'foo':self.foo,}) + + * Sample usage + + .. code-block:: python + + >>> import jsonrpclib + >>> import test_obj + + # History is used only to print the serialized form of beans + >>> history = jsonrpclib.history.History() + >>> testobj1 = test_obj.TestObj() + >>> testobj2 = test_obj.TestSerial() + >>> server = jsonrpclib.Server('http://localhost:8080', history=history) + + # The 'ping' just returns whatever is sent + >>> ping1 = server.ping(testobj1) + >>> ping2 = server.ping(testobj2) + + >>> print(history.request) + {"id": "7805f1f9-9abd-49c6-81dc-dbd47229fe13", "jsonrpc": "2.0", + "method": "ping", "params": [{"__jsonclass__": + ["test_obj.TestSerial", []], "foo": "bar"} + ]} + >>> print(history.response) + {"id": "7805f1f9-9abd-49c6-81dc-dbd47229fe13", "jsonrpc": "2.0", + "result": {"__jsonclass__": ["test_obj.TestSerial", []], "foo": "bar"}} + + This behavior is turned by default. To deactivate it, just set the + ``use_jsonclass`` member of a server ``Config`` to False. + If you want to use a per-class serialization method, set its name in the + ``serialize_method`` member of a server ``Config``. + Finally, if you are using classes that you have defined in the implementation + (as in, not a separate library), you'll need to add those (on BOTH the server + and the client) using the ``config.classes.add()`` method. + + Feedback on this "feature" is very, VERY much appreciated. + + Why JSON-RPC? + ************* + + In my opinion, there are several reasons to choose JSON over XML for RPC: + + * Much simpler to read (I suppose this is opinion, but I know I'm right. :) + * Size / Bandwidth - Main reason, a JSON object representation is just much smaller. + * Parsing - JSON should be much quicker to parse than XML. + * Easy class passing with ``jsonclass`` (when enabled) + + In the interest of being fair, there are also a few reasons to choose XML + over JSON: + + * Your server doesn't do JSON (rather obvious) + * Wider XML-RPC support across APIs (can we change this? :)) + * Libraries are more established, i.e. more stable (Let's change this too.) + + Tests + ***** + + Tests are an almost-verbatim drop from the JSON-RPC specification 2.0 page. + They can be run using *unittest* or *nosetest*: + + .. code-block:: console + + python -m unittest discover tests + python3 -m unittest discover tests + nosetests tests + +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.0 +Classifier: Programming Language :: Python :: 3.1 +Classifier: Programming Language :: Python :: 3.2 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/SOURCES.txt b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/SOURCES.txt new file mode 100644 index 00000000..f5714032 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/SOURCES.txt @@ -0,0 +1,17 @@ +LICENSE.txt +MANIFEST.in +README.rst +setup.cfg +setup.py +jsonrpclib/SimpleJSONRPCServer.py +jsonrpclib/__init__.py +jsonrpclib/config.py +jsonrpclib/history.py +jsonrpclib/jsonclass.py +jsonrpclib/jsonrpc.py +jsonrpclib/threadpool.py +jsonrpclib/utils.py +jsonrpclib_pelix.egg-info/PKG-INFO +jsonrpclib_pelix.egg-info/SOURCES.txt +jsonrpclib_pelix.egg-info/dependency_links.txt +jsonrpclib_pelix.egg-info/top_level.txt
\ No newline at end of file diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/dependency_links.txt b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/top_level.txt b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/top_level.txt new file mode 100644 index 00000000..1410b2ff --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/jsonrpclib_pelix.egg-info/top_level.txt @@ -0,0 +1 @@ +jsonrpclib diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/setup.cfg b/scripts/external_libs/jsonrpclib-pelix-0.2.5/setup.cfg new file mode 100644 index 00000000..7633f817 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/setup.cfg @@ -0,0 +1,8 @@ +[bdist_wheel] +universal = 1 + +[egg_info] +tag_date = 0 +tag_svn_revision = 0 +tag_build = + diff --git a/scripts/external_libs/jsonrpclib-pelix-0.2.5/setup.py b/scripts/external_libs/jsonrpclib-pelix-0.2.5/setup.py new file mode 100644 index 00000000..fb28d630 --- /dev/null +++ b/scripts/external_libs/jsonrpclib-pelix-0.2.5/setup.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# -- Content-Encoding: UTF-8 -- +""" +Installation script + +:authors: Josh Marshall, Thomas Calmant +:copyright: Copyright 2015, isandlaTech +:license: Apache License 2.0 +:version: 0.2.5 + +.. + + Copyright 2015 isandlaTech + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +# Module version +__version_info__ = (0, 2, 5) +__version__ = ".".join(str(x) for x in __version_info__) + +# Documentation strings format +__docformat__ = "restructuredtext en" + +# ------------------------------------------------------------------------------ + +import sys + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +# ------------------------------------------------------------------------------ + +setup( + name="jsonrpclib-pelix", + version=__version__, + license="Apache License 2.0", + author="Thomas Calmant", + author_email="thomas.calmant+github@gmail.com", + url="http://github.com/tcalmant/jsonrpclib/", + description= + "This project is an implementation of the JSON-RPC v2.0 specification " + "(backwards-compatible) as a client library, for Python 2.6+ and Python 3." + "This version is a fork of jsonrpclib by Josh Marshall, " + "usable with Pelix remote services.", + long_description=open("README.rst").read(), + packages=["jsonrpclib"], + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.0', + 'Programming Language :: Python :: 3.1', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4'], + tests_require=['unittest2'] if sys.version_info < (2, 7) else [] +) diff --git a/scripts/external_libs/lockfile-0.10.2/ACKS b/scripts/external_libs/lockfile-0.10.2/ACKS new file mode 100644 index 00000000..44519d17 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/ACKS @@ -0,0 +1,6 @@ +Thanks to the following people for help with lockfile. + + Scott Dial + Ben Finney + Frank Niessink + Konstantin Veretennicov diff --git a/scripts/external_libs/lockfile-0.10.2/AUTHORS b/scripts/external_libs/lockfile-0.10.2/AUTHORS new file mode 100644 index 00000000..fda721cd --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/AUTHORS @@ -0,0 +1,5 @@ +(no author) <(no author)> +Elmo Todurov <elmo.todurov@skype.net> +Julien Danjou <julien@danjou.info> +Skip Montanaro <skip@pobox.com> +skip.montanaro <skip.montanaro@gmail.com> diff --git a/scripts/external_libs/lockfile-0.10.2/ChangeLog b/scripts/external_libs/lockfile-0.10.2/ChangeLog new file mode 100644 index 00000000..3ba36a7d --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/ChangeLog @@ -0,0 +1,165 @@ +CHANGES +======= + +0.10.2 +------ + +* Fix package name + +0.10.1 +------ + +* Add missing cover env in tox + +0.10.0 +------ + +* Fix documentation bug report address +* Add py34 in tox +* Remove old diff file +* Add .gitreview, tox targets and use pbr +* fix for timeout=0 +* remove 2.5, 3.1 and 3.4 from the list for the time being - may get added back later +* Bugfix: locking two different files in the same directory caused an error during unlocking the last unlocking +* typo + +0.9.1 +----- + +* ignore dist dir +* update to python 3 imports +* python 3 tweaks +* python 3 tweaks +* ignore Emacs backups +* note nose as a dependency +* remove this test file - way incompatible with current code +* stuff to ignore +* Add py33, py34, delete py24, py25 +* Update source location +* merge delete +* merge delete +* more merge stuff +* this didn't come across with svn merge +* all screwed up now +* merge +* merge +* Make it clear that the path and threaded attributes to SymlinkLockFile and MkdirLockFile have the same constraints as for LinkLockFile. In particular, the directory which will contain path must be writable +* add pidlockfile test stuff from Ben Finney - still a few problems - maybe I can get him to solve them :-) +* ignore Sphinx build directory +* Catch up on a little documentation +* adapt decorator patch from issue 5 +* Allow timeout in constructor - resolves issue 3 +* add info to raise statements - from issue 6, yyurevich@jellycrystal.com +* add useful repr() - from issue 6, yyurevich@jellycrystal.com +* add symlinklockfile module +* + py24 +* good for the branch? must be good for the trunk +* add tox stuff, ignore dist dir +* new version, move to Google Code +* +* +* * Thread support is currently broken. This is more likely because of problems in this module, but suppress those tests for now just the same +* By the nature of what it's trying to do PIDLockFile doesn't support threaded operation +* defer creating testdb until we've instantiated a SQLiteLockFile instance +* tweak unique_name slightly +* Specify mode in octal +* update to match pidlockfile change +* missing import +* I think I finally have this correct +* patch pidlockfile module too +* use abs import here as well +* *argh* +* Update to elide new import syntax +* * Move future import for division where it's used. * Use __absolute_import__ to spell relative imports +* Some PIDLockFile tests are failing. Check in anyway so others can consider the problems +* Account for fact that Thread objects in Python 2.4 and earlier do not have an ident attribute +* Make this a daemon thread so if things go awry the test run won't hang +* * Add pidlockfile (not quite working properly) * Rearrange MANIFEST.in slightly to include test directory +* Split those test methods which try both threaded and non-threaded naming schemes. More to do. Obviously you need to have test cases when using the non-threaded naming scheme from multiple threads +* acknowledge Ben and Frank, alphabetize list +* I don't think these are needed any longer - they came back during the hg->svn conversion +* grand renaming: "filelock" -> "lockfile" & "FileLock" -> "LockFile" +* Update for packages +* Avoid using the backwards compatibility functions for FileLock. That object is not deprecated +* how does the test dir keep sneaking into MANIFEST? also, include 2.4.diff in dist +* update for new structure, use of ident attr +* adjust build setup +* move test helpers into test dir +* first cut at packagized lockfile +* Protect some more complex locking stuff so if they fail we don't deadlock +* merge r75 from head +* * One implementation of tname, not two - make it an instance attribute as a result +* beginnings of a packagized lockfile +* get the structure right +* start over with the branches.. +* hmmm +* hmmm +* get us back to lockfile 0.8 +* r72 from hg +* r70 from hg +* r69 from hg +* r68 from hg +* r67 from hg +* r66 from hg +* r65 from hg +* r64 from hg +* r64 from hg +* r63 from hg +* r62 from hg +* r61 from hg +* r60 from hg +* r59 from hg +* r58 from hg +* r57 from hg +* r56 from hg +* r55 from hg +* r54 from hg +* r53 from hg +* r52 from hg +* r51 from hg +* r50 from hg +* r49 from hg +* r47 from hg +* r46 from hg +* r45 from hg +* r44 from hg +* r43 from hg +* r42 from hg +* r41 from hg +* r38 from hg +* r37 from hg +* r36 from hg +* r35 from hg +* r34 from hg +* r33 from hg +* r32 from hg +* r31 from hg +* r29 from hg +* r28 from hg +* r27 from hg +* r26 from hg +* r25 from hg +* r24 from hg +* r23 from hg +* r22 from hg +* r21 from hg +* r20 from hg +* r19 from hg +* r18 from hg +* r16 from hg +* r14 from hg +* r13 from hg +* r12 from hg +* r11 from hg +* r10 from hg +* r9 from hg +* r8 from hg +* r7 from hg +* r6 from hg +* r5 from hg +* r4 from hg +* r3 from hg +* r2 from hg +* r1 from hg +* r0 from hg +* Initial directory structure diff --git a/scripts/external_libs/lockfile-0.10.2/LICENSE b/scripts/external_libs/lockfile-0.10.2/LICENSE new file mode 100644 index 00000000..610c0793 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/LICENSE @@ -0,0 +1,21 @@ +This is the MIT license: http://www.opensource.org/licenses/mit-license.php + +Copyright (c) 2007 Skip Montanaro. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/scripts/external_libs/lockfile-0.10.2/PKG-INFO b/scripts/external_libs/lockfile-0.10.2/PKG-INFO new file mode 100644 index 00000000..9f72376f --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/PKG-INFO @@ -0,0 +1,51 @@ +Metadata-Version: 1.1 +Name: lockfile +Version: 0.10.2 +Summary: Platform-independent file locking module +Home-page: http://launchpad.net/pylockfile +Author: OpenStack +Author-email: openstack-dev@lists.openstack.org +License: UNKNOWN +Description: The lockfile package exports a LockFile class which provides a simple API for + locking files. Unlike the Windows msvcrt.locking function, the fcntl.lockf + and flock functions, and the deprecated posixfile module, the API is + identical across both Unix (including Linux and Mac) and Windows platforms. + The lock mechanism relies on the atomic nature of the link (on Unix) and + mkdir (on Windows) system calls. An implementation based on SQLite is also + provided, more as a demonstration of the possibilities it provides than as + production-quality code. + + Note: In version 0.9 the API changed in two significant ways: + + * It changed from a module defining several classes to a package containing + several modules, each defining a single class. + + * Where classes had been named SomethingFileLock before the last two words + have been reversed, so that class is now SomethingLockFile. + + The previous module-level definitions of LinkFileLock, MkdirFileLock and + SQLiteFileLock will be retained until the 1.0 release. + + Available on GitHub from: + + git://github.com/smontanaro/pylockfile.git + + To install: + + python setup.py install + + +Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: POSIX :: Linux +Classifier: Operating System :: MacOS +Classifier: Operating System :: Microsoft :: Windows :: Windows NT/2000 +Classifier: Operating System :: POSIX +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Topic :: Software Development :: Libraries :: Python Modules diff --git a/scripts/external_libs/lockfile-0.10.2/README b/scripts/external_libs/lockfile-0.10.2/README new file mode 100644 index 00000000..5f7acbc4 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/README @@ -0,0 +1,27 @@ +The lockfile package exports a LockFile class which provides a simple API for +locking files. Unlike the Windows msvcrt.locking function, the fcntl.lockf +and flock functions, and the deprecated posixfile module, the API is +identical across both Unix (including Linux and Mac) and Windows platforms. +The lock mechanism relies on the atomic nature of the link (on Unix) and +mkdir (on Windows) system calls. An implementation based on SQLite is also +provided, more as a demonstration of the possibilities it provides than as +production-quality code. + +Note: In version 0.9 the API changed in two significant ways: + + * It changed from a module defining several classes to a package containing + several modules, each defining a single class. + + * Where classes had been named SomethingFileLock before the last two words + have been reversed, so that class is now SomethingLockFile. + +The previous module-level definitions of LinkFileLock, MkdirFileLock and +SQLiteFileLock will be retained until the 1.0 release. + +Available on GitHub from: + + git://github.com/smontanaro/pylockfile.git + +To install: + + python setup.py install diff --git a/scripts/external_libs/lockfile-0.10.2/RELEASE-NOTES b/scripts/external_libs/lockfile-0.10.2/RELEASE-NOTES new file mode 100644 index 00000000..8b452ed1 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/RELEASE-NOTES @@ -0,0 +1,50 @@ +Version 0.9.1 +============= + +* This release moves the source location to Google Code. + +* Threaded support is currently broken. (It might not actually be broken. + It might just be the tests which are broken.) + +Version 0.9 +=========== + +* The lockfile module was reorganized into a package. + +* The names of the three main classes have changed as follows: + + LinkFileLock -> LinkLockFile + MkdirFileLock -> MkdirLockFile + SQLiteFileLock -> SQLiteLockFile + +* A PIDLockFile class was added. + +Version 0.3 +=========== + +* Fix 2.4.diff file error. + +* More documentation updates. + +Version 0.2 +=========== + +* Added 2.4.diff file to patch lockfile to work with Python 2.4 (removes use + of with statement). + +* Renamed _FileLock base class to LockBase to expose it (and its docstrings) + to pydoc. + +* Got rid of time.sleep() calls in tests (thanks to Konstantin + Veretennicov). + +* Use thread.get_ident() as the thread discriminator. + +* Updated documentation a bit. + +* Added RELEASE-NOTES. + +Version 0.1 +=========== + +* First release - All basic functionality there. diff --git a/scripts/external_libs/lockfile-0.10.2/doc/source/Makefile b/scripts/external_libs/lockfile-0.10.2/doc/source/Makefile new file mode 100644 index 00000000..1b1e8d28 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/doc/source/Makefile @@ -0,0 +1,73 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d .build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html web pickle htmlhelp latex changes linkcheck + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " pickle to make pickle files (usable by e.g. sphinx-web)" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview over all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + +clean: + -rm -rf .build/* + +html: + mkdir -p .build/html .build/doctrees + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) .build/html + @echo + @echo "Build finished. The HTML pages are in .build/html." + +pickle: + mkdir -p .build/pickle .build/doctrees + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) .build/pickle + @echo + @echo "Build finished; now you can process the pickle files or run" + @echo " sphinx-web .build/pickle" + @echo "to start the sphinx-web server." + +web: pickle + +htmlhelp: + mkdir -p .build/htmlhelp .build/doctrees + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) .build/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in .build/htmlhelp." + +latex: + mkdir -p .build/latex .build/doctrees + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) .build/latex + @echo + @echo "Build finished; the LaTeX files are in .build/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + mkdir -p .build/changes .build/doctrees + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) .build/changes + @echo + @echo "The overview file is in .build/changes." + +linkcheck: + mkdir -p .build/linkcheck .build/doctrees + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) .build/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in .build/linkcheck/output.txt." + +html.zip: html + (cd .build/html ; zip -r ../../$@ *) diff --git a/scripts/external_libs/lockfile-0.10.2/doc/source/conf.py b/scripts/external_libs/lockfile-0.10.2/doc/source/conf.py new file mode 100644 index 00000000..623edcb5 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/doc/source/conf.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +# +# lockfile documentation build configuration file, created by +# sphinx-quickstart on Sat Sep 13 17:54:17 2008. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# The contents of this file are pickled, so don't put values in the namespace +# that aren't pickleable (module imports are okay, they're removed automatically). +# +# All configuration values have a default value; values that are commented out +# serve to show the default value. + +import sys, os + +# If your extensions are in another directory, add it here. If the directory +# is relative to the documentation root, use os.path.abspath to make it +# absolute, like shown here. +#sys.path.append(os.path.abspath('some/directory')) + +# General configuration +# --------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General substitutions. +project = 'lockfile' +copyright = '2008, Skip Montanaro' + +# The default replacements for |version| and |release|, also used in various +# other places throughout the built documents. +# +# The short X.Y version. +version = '0.3' +# The full version, including alpha/beta/rc tags. +release = '0.3' + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directories, that shouldn't be searched +# for source files. +#exclude_dirs = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# Options for HTML output +# ----------------------- + +# The style sheet to use for HTML and HTML Help pages. A file of that name +# must exist either in Sphinx' static/ path, or in one of the custom paths +# given in html_static_path. +html_style = 'default.css' + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (within the static path) to place at the top of +# the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['.static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, the reST sources are included in the HTML build as _sources/<name>. +#html_copy_source = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'lockfiledoc' + + +# Options for LaTeX output +# ------------------------ + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, document class [howto/manual]). +latex_documents = [ + ('lockfile', 'lockfile.tex', 'lockfile Documentation', + 'Skip Montanaro', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/scripts/external_libs/lockfile-0.10.2/doc/source/index.rst b/scripts/external_libs/lockfile-0.10.2/doc/source/index.rst new file mode 100644 index 00000000..f76173dc --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/doc/source/index.rst @@ -0,0 +1,275 @@ + +:mod:`lockfile` --- Platform-independent file locking +===================================================== + +.. module:: lockfile + :synopsis: Platform-independent file locking +.. moduleauthor:: Skip Montanaro <skip@pobox.com> +.. sectionauthor:: Skip Montanaro <skip@pobox.com> + + +.. note:: + + This package is pre-release software. Between versions 0.8 and 0.9 it + was changed from a module to a package. It is quite possible that the + API and implementation will change again in important ways as people test + it and provide feedback and bug fixes. In particular, if the mkdir-based + locking scheme is sufficient for both Windows and Unix platforms, the + link-based scheme may be deleted so that only a single locking scheme is + used, providing cross-platform lockfile cooperation. + +.. note:: + + The implementation uses the `with` statement, both in the tests and in the + main code, so will only work out-of-the-box with Python 2.5 or later. + However, the use of the `with` statement is minimal, so if you apply the + patch in the included 2.4.diff file you can use it with Python 2.4. It's + possible that it will work in Python 2.3 with that patch applied as well, + though the doctest code relies on APIs new in 2.4, so will have to be + rewritten somewhat to allow testing on 2.3. As they say, patches welcome. + ``;-)`` + +The :mod:`lockfile` package exports a :class:`LockFile` class which provides +a simple API for locking files. Unlike the Windows :func:`msvcrt.locking` +function, the Unix :func:`fcntl.flock`, :func:`fcntl.lockf` and the +deprecated :mod:`posixfile` module, the API is identical across both Unix +(including Linux and Mac) and Windows platforms. The lock mechanism relies +on the atomic nature of the :func:`link` (on Unix) and :func:`mkdir` (On +Windows) system calls. It also contains several lock-method-specific +modules: :mod:`lockfile.linklockfile`, :mod:`lockfile.mkdirlockfile`, and +:mod:`lockfile.sqlitelockfile`, each one exporting a single class. For +backwards compatibility with versions before 0.9 the :class:`LinkFileLock`, +:class:`MkdirFileLock` and :class:`SQLiteFileLock` objects are exposed as +attributes of the top-level lockfile package, though this use was deprecated +starting with version 0.9 and will be removed in version 1.0. + +.. note:: + + The current implementation uses :func:`os.link` on Unix, but since that + function is unavailable on Windows it uses :func:`os.mkdir` there. At + this point it's not clear that using the :func:`os.mkdir` method would be + insufficient on Unix systems. If it proves to be adequate on Unix then + the implementation could be simplified and truly cross-platform locking + would be possible. + +.. note:: + + The current implementation doesn't provide for shared vs. exclusive + locks. It should be possible for multiple reader processes to hold the + lock at the same time. + +The module defines the following exceptions: + +.. exception:: Error + + This is the base class for all exceptions raised by the :class:`LockFile` + class. + +.. exception:: LockError + + This is the base class for all exceptions raised when attempting to lock + a file. + +.. exception:: UnlockError + + This is the base class for all exceptions raised when attempting to + unlock a file. + +.. exception:: LockTimeout + + This exception is raised if the :func:`LockFile.acquire` method is + called with a timeout which expires before an existing lock is released. + +.. exception:: AlreadyLocked + + This exception is raised if the :func:`LockFile.acquire` detects a + file is already locked when in non-blocking mode. + +.. exception:: LockFailed + + This exception is raised if the :func:`LockFile.acquire` detects some + other condition (such as a non-writable directory) which prevents it from + creating its lock file. + +.. exception:: NotLocked + + This exception is raised if the file is not locked when + :func:`LockFile.release` is called. + +.. exception:: NotMyLock + + This exception is raised if the file is locked by another thread or + process when :func:`LockFile.release` is called. + +The following classes are provided: + +.. class:: linklockfile.LinkLockFile(path, threaded=True) + + This class uses the :func:`link(2)` system call as the basic lock + mechanism. *path* is an object in the file system to be locked. It need + not exist, but its directory must exist and be writable at the time the + :func:`acquire` and :func:`release` methods are called. *threaded* is + optional, but when set to :const:`True` locks will be distinguished + between threads in the same process. + +.. class:: symlinklockfile.SymlinkLockFile(path, threaded=True) + + This class uses the :func:`symlink(2)` system call as the basic lock + mechanism. The parameters have the same meaning and constraints as for + the :class:`LinkLockFile` class. + +.. class:: mkdirlockfile.MkdirLockFile(path, threaded=True) + + This class uses the :func:`mkdir(2)` system call as the basic lock + mechanism. The parameters have the same meaning and constraints as for + the :class:`LinkLockFile` class. + +.. class:: sqlitelockfile.SQLiteLockFile(path, threaded=True) + + This class uses the :mod:`sqlite3` module to implement the lock + mechanism. The parameters have the same meaning as for the + :class:`LinkLockFile` class. + +.. class:: LockBase(path, threaded=True) + + This is the base class for all concrete implementations and is available + at the lockfile package level so programmers can implement other locking + schemes. + +.. function:: locked(path, timeout=None) + + This function provides a decorator which insures the decorated function + is always called with the lock held. + +By default, the :const:`LockFile` object refers to the +:class:`mkdirlockfile.MkdirLockFile` class on Windows. On all other +platforms it refers to the :class:`linklockfile.LinkLockFile` class. + +When locking a file the :class:`linklockfile.LinkLockFile` class creates a +uniquely named hard link to an empty lock file. That hard link contains the +hostname, process id, and if locks between threads are distinguished, the +thread identifier. For example, if you want to lock access to a file named +"README", the lock file is named "README.lock". With per-thread locks +enabled the hard link is named HOSTNAME-THREADID-PID. With only per-process +locks enabled the hard link is named HOSTNAME--PID. + +When using the :class:`mkdirlockfile.MkdirLockFile` class the lock file is a +directory. Referring to the example above, README.lock will be a directory +and HOSTNAME-THREADID-PID will be an empty file within that directory. + +.. seealso:: + + Module :mod:`msvcrt` + Provides the :func:`locking` function, the standard Windows way of + locking (parts of) a file. + + Module :mod:`posixfile` + The deprecated (since Python 1.5) way of locking files on Posix systems. + + Module :mod:`fcntl` + Provides the current best way to lock files on Unix systems + (:func:`lockf` and :func:`flock`). + +LockFile Objects +---------------- + +:class:`LockFile` objects support the `context manager` protocol used by the +statement:`with` statement. The timeout option is not supported when used in +this fashion. While support for timeouts could be implemented, there is no +support for handling the eventual :exc:`Timeout` exceptions raised by the +:func:`__enter__` method, so you would have to protect the `with` statement with +a `try` statement. The resulting construct would not be any simpler than just +using a `try` statement in the first place. + +:class:`LockFile` has the following user-visible methods: + +.. method:: LockFile.acquire(timeout=None) + + Lock the file associated with the :class:`LockFile` object. If the + *timeout* is omitted or :const:`None` the caller will block until the + file is unlocked by the object currently holding the lock. If the + *timeout* is zero or a negative number the :exc:`AlreadyLocked` exception + will be raised if the file is currently locked by another process or + thread. If the *timeout* is positive, the caller will block for that + many seconds waiting for the lock to be released. If the lock is not + released within that period the :exc:`LockTimeout` exception will be + raised. + +.. method:: LockFile.release() + + Unlock the file associated with the :class:`LockFile` object. If the + file is not currently locked, the :exc:`NotLocked` exception is raised. + If the file is locked by another thread or process the :exc:`NotMyLock` + exception is raised. + +.. method:: is_locked() + + Return the status of the lock on the current file. If any process or + thread (including the current one) is locking the file, :const:`True` is + returned, otherwise :const:`False` is returned. + +.. method:: break_lock() + + If the file is currently locked, break it. + +.. method:: i_am_locking() + + Returns true if the caller holds the lock. + +Examples +-------- + +This example is the "hello world" for the :mod:`lockfile` package:: + + from lockfile import LockFile + lock = LockFile("/some/file/or/other") + with lock: + print lock.path, 'is locked.' + +To use this with Python 2.4, you can execute:: + + from lockfile import LockFile + lock = LockFile("/some/file/or/other") + lock.acquire() + print lock.path, 'is locked.' + lock.release() + +If you don't want to wait forever, you might try:: + + from lockfile import LockFile + lock = LockFile("/some/file/or/other") + while not lock.i_am_locking(): + try: + lock.acquire(timeout=60) # wait up to 60 seconds + except LockTimeout: + lock.break_lock() + lock.acquire() + print "I locked", lock.path + lock.release() + +You can also insure that a lock is always held when appropriately decorated +functions are called:: + + from lockfile import locked + @locked("/tmp/mylock") + def func(a, b): + return a + b + +Other Libraries +--------------- + +The idea of implementing advisory locking with a standard API is not new +with :mod:`lockfile`. There are a number of other libraries available: + +* locknix - http://pypi.python.org/pypi/locknix - Unix only +* mx.MiscLockFile - from Marc André Lemburg, part of the mx.Base + distribution - cross-platform. +* Twisted - http://twistedmatrix.com/trac/browser/trunk/twisted/python/lockfile.py +* zc.lockfile - http://pypi.python.org/pypi/zc.lockfile + + +Contacting the Author +--------------------- + +If you encounter any problems with ``lockfile``, would like help or want to +submit a patch, check http://launchpad.net/pylockfile diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/PKG-INFO b/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/PKG-INFO new file mode 100644 index 00000000..9f72376f --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/PKG-INFO @@ -0,0 +1,51 @@ +Metadata-Version: 1.1 +Name: lockfile +Version: 0.10.2 +Summary: Platform-independent file locking module +Home-page: http://launchpad.net/pylockfile +Author: OpenStack +Author-email: openstack-dev@lists.openstack.org +License: UNKNOWN +Description: The lockfile package exports a LockFile class which provides a simple API for + locking files. Unlike the Windows msvcrt.locking function, the fcntl.lockf + and flock functions, and the deprecated posixfile module, the API is + identical across both Unix (including Linux and Mac) and Windows platforms. + The lock mechanism relies on the atomic nature of the link (on Unix) and + mkdir (on Windows) system calls. An implementation based on SQLite is also + provided, more as a demonstration of the possibilities it provides than as + production-quality code. + + Note: In version 0.9 the API changed in two significant ways: + + * It changed from a module defining several classes to a package containing + several modules, each defining a single class. + + * Where classes had been named SomethingFileLock before the last two words + have been reversed, so that class is now SomethingLockFile. + + The previous module-level definitions of LinkFileLock, MkdirFileLock and + SQLiteFileLock will be retained until the 1.0 release. + + Available on GitHub from: + + git://github.com/smontanaro/pylockfile.git + + To install: + + python setup.py install + + +Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: POSIX :: Linux +Classifier: Operating System :: MacOS +Classifier: Operating System :: Microsoft :: Windows :: Windows NT/2000 +Classifier: Operating System :: POSIX +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Topic :: Software Development :: Libraries :: Python Modules diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/SOURCES.txt b/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/SOURCES.txt new file mode 100644 index 00000000..4b289f3a --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/SOURCES.txt @@ -0,0 +1,26 @@ +ACKS +AUTHORS +ChangeLog +LICENSE +README +RELEASE-NOTES +setup.cfg +setup.py +test-requirements.txt +tox.ini +doc/source/Makefile +doc/source/conf.py +doc/source/index.rst +lockfile/__init__.py +lockfile/linklockfile.py +lockfile/mkdirlockfile.py +lockfile/pidlockfile.py +lockfile/sqlitelockfile.py +lockfile/symlinklockfile.py +lockfile.egg-info/PKG-INFO +lockfile.egg-info/SOURCES.txt +lockfile.egg-info/dependency_links.txt +lockfile.egg-info/not-zip-safe +lockfile.egg-info/top_level.txt +test/compliancetest.py +test/test_lockfile.py
\ No newline at end of file diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/dependency_links.txt b/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/not-zip-safe b/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/not-zip-safe new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/top_level.txt b/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/top_level.txt new file mode 100644 index 00000000..5a13159a --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile.egg-info/top_level.txt @@ -0,0 +1 @@ +lockfile diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile/__init__.py b/scripts/external_libs/lockfile-0.10.2/lockfile/__init__.py new file mode 100644 index 00000000..d905af96 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile/__init__.py @@ -0,0 +1,326 @@ +""" +lockfile.py - Platform-independent advisory file locks. + +Requires Python 2.5 unless you apply 2.4.diff +Locking is done on a per-thread basis instead of a per-process basis. + +Usage: + +>>> lock = LockFile('somefile') +>>> try: +... lock.acquire() +... except AlreadyLocked: +... print 'somefile', 'is locked already.' +... except LockFailed: +... print 'somefile', 'can\\'t be locked.' +... else: +... print 'got lock' +got lock +>>> print lock.is_locked() +True +>>> lock.release() + +>>> lock = LockFile('somefile') +>>> print lock.is_locked() +False +>>> with lock: +... print lock.is_locked() +True +>>> print lock.is_locked() +False + +>>> lock = LockFile('somefile') +>>> # It is okay to lock twice from the same thread... +>>> with lock: +... lock.acquire() +... +>>> # Though no counter is kept, so you can't unlock multiple times... +>>> print lock.is_locked() +False + +Exceptions: + + Error - base class for other exceptions + LockError - base class for all locking exceptions + AlreadyLocked - Another thread or process already holds the lock + LockFailed - Lock failed for some other reason + UnlockError - base class for all unlocking exceptions + AlreadyUnlocked - File was not locked. + NotMyLock - File was locked but not by the current thread/process +""" + +from __future__ import absolute_import + +import sys +import socket +import os +import threading +import time +import urllib +import warnings +import functools + +# Work with PEP8 and non-PEP8 versions of threading module. +if not hasattr(threading, "current_thread"): + threading.current_thread = threading.currentThread +if not hasattr(threading.Thread, "get_name"): + threading.Thread.get_name = threading.Thread.getName + +__all__ = ['Error', 'LockError', 'LockTimeout', 'AlreadyLocked', + 'LockFailed', 'UnlockError', 'NotLocked', 'NotMyLock', + 'LinkLockFile', 'MkdirLockFile', 'SQLiteLockFile', + 'LockBase', 'locked'] + +class Error(Exception): + """ + Base class for other exceptions. + + >>> try: + ... raise Error + ... except Exception: + ... pass + """ + pass + +class LockError(Error): + """ + Base class for error arising from attempts to acquire the lock. + + >>> try: + ... raise LockError + ... except Error: + ... pass + """ + pass + +class LockTimeout(LockError): + """Raised when lock creation fails within a user-defined period of time. + + >>> try: + ... raise LockTimeout + ... except LockError: + ... pass + """ + pass + +class AlreadyLocked(LockError): + """Some other thread/process is locking the file. + + >>> try: + ... raise AlreadyLocked + ... except LockError: + ... pass + """ + pass + +class LockFailed(LockError): + """Lock file creation failed for some other reason. + + >>> try: + ... raise LockFailed + ... except LockError: + ... pass + """ + pass + +class UnlockError(Error): + """ + Base class for errors arising from attempts to release the lock. + + >>> try: + ... raise UnlockError + ... except Error: + ... pass + """ + pass + +class NotLocked(UnlockError): + """Raised when an attempt is made to unlock an unlocked file. + + >>> try: + ... raise NotLocked + ... except UnlockError: + ... pass + """ + pass + +class NotMyLock(UnlockError): + """Raised when an attempt is made to unlock a file someone else locked. + + >>> try: + ... raise NotMyLock + ... except UnlockError: + ... pass + """ + pass + +class LockBase: + """Base class for platform-specific lock classes.""" + def __init__(self, path, threaded=True, timeout=None): + """ + >>> lock = LockBase('somefile') + >>> lock = LockBase('somefile', threaded=False) + """ + self.path = path + self.lock_file = os.path.abspath(path) + ".lock" + self.hostname = socket.gethostname() + self.pid = os.getpid() + if threaded: + t = threading.current_thread() + # Thread objects in Python 2.4 and earlier do not have ident + # attrs. Worm around that. + ident = getattr(t, "ident", hash(t)) + self.tname = "-%x" % (ident & 0xffffffff) + else: + self.tname = "" + dirname = os.path.dirname(self.lock_file) + + # unique name is mostly about the current process, but must + # also contain the path -- otherwise, two adjacent locked + # files conflict (one file gets locked, creating lock-file and + # unique file, the other one gets locked, creating lock-file + # and overwriting the already existing lock-file, then one + # gets unlocked, deleting both lock-file and unique file, + # finally the last lock errors out upon releasing. + self.unique_name = os.path.join(dirname, + "%s%s.%s%s" % (self.hostname, + self.tname, + self.pid, + hash(self.path))) + self.timeout = timeout + + def acquire(self, timeout=None): + """ + Acquire the lock. + + * If timeout is omitted (or None), wait forever trying to lock the + file. + + * If timeout > 0, try to acquire the lock for that many seconds. If + the lock period expires and the file is still locked, raise + LockTimeout. + + * If timeout <= 0, raise AlreadyLocked immediately if the file is + already locked. + """ + raise NotImplemented("implement in subclass") + + def release(self): + """ + Release the lock. + + If the file is not locked, raise NotLocked. + """ + raise NotImplemented("implement in subclass") + + def is_locked(self): + """ + Tell whether or not the file is locked. + """ + raise NotImplemented("implement in subclass") + + def i_am_locking(self): + """ + Return True if this object is locking the file. + """ + raise NotImplemented("implement in subclass") + + def break_lock(self): + """ + Remove a lock. Useful if a locking thread failed to unlock. + """ + raise NotImplemented("implement in subclass") + + def __enter__(self): + """ + Context manager support. + """ + self.acquire() + return self + + def __exit__(self, *_exc): + """ + Context manager support. + """ + self.release() + + def __repr__(self): + return "<%s: %r -- %r>" % (self.__class__.__name__, self.unique_name, + self.path) + +def _fl_helper(cls, mod, *args, **kwds): + warnings.warn("Import from %s module instead of lockfile package" % mod, + DeprecationWarning, stacklevel=2) + # This is a bit funky, but it's only for awhile. The way the unit tests + # are constructed this function winds up as an unbound method, so it + # actually takes three args, not two. We want to toss out self. + if not isinstance(args[0], str): + # We are testing, avoid the first arg + args = args[1:] + if len(args) == 1 and not kwds: + kwds["threaded"] = True + return cls(*args, **kwds) + +def LinkFileLock(*args, **kwds): + """Factory function provided for backwards compatibility. + + Do not use in new code. Instead, import LinkLockFile from the + lockfile.linklockfile module. + """ + from . import linklockfile + return _fl_helper(linklockfile.LinkLockFile, "lockfile.linklockfile", + *args, **kwds) + +def MkdirFileLock(*args, **kwds): + """Factory function provided for backwards compatibility. + + Do not use in new code. Instead, import MkdirLockFile from the + lockfile.mkdirlockfile module. + """ + from . import mkdirlockfile + return _fl_helper(mkdirlockfile.MkdirLockFile, "lockfile.mkdirlockfile", + *args, **kwds) + +def SQLiteFileLock(*args, **kwds): + """Factory function provided for backwards compatibility. + + Do not use in new code. Instead, import SQLiteLockFile from the + lockfile.mkdirlockfile module. + """ + from . import sqlitelockfile + return _fl_helper(sqlitelockfile.SQLiteLockFile, "lockfile.sqlitelockfile", + *args, **kwds) + +def locked(path, timeout=None): + """Decorator which enables locks for decorated function. + + Arguments: + - path: path for lockfile. + - timeout (optional): Timeout for acquiring lock. + + Usage: + @locked('/var/run/myname', timeout=0) + def myname(...): + ... + """ + def decor(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + lock = FileLock(path, timeout=timeout) + lock.acquire() + try: + return func(*args, **kwargs) + finally: + lock.release() + return wrapper + return decor + +if hasattr(os, "link"): + from . import linklockfile as _llf + LockFile = _llf.LinkLockFile +else: + from . import mkdirlockfile as _mlf + LockFile = _mlf.MkdirLockFile + +FileLock = LockFile + diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile/linklockfile.py b/scripts/external_libs/lockfile-0.10.2/lockfile/linklockfile.py new file mode 100644 index 00000000..9c506734 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile/linklockfile.py @@ -0,0 +1,73 @@ +from __future__ import absolute_import + +import time +import os + +from . import (LockBase, LockFailed, NotLocked, NotMyLock, LockTimeout, + AlreadyLocked) + +class LinkLockFile(LockBase): + """Lock access to a file using atomic property of link(2). + + >>> lock = LinkLockFile('somefile') + >>> lock = LinkLockFile('somefile', threaded=False) + """ + + def acquire(self, timeout=None): + try: + open(self.unique_name, "wb").close() + except IOError: + raise LockFailed("failed to create %s" % self.unique_name) + + timeout = timeout is not None and timeout or self.timeout + end_time = time.time() + if timeout is not None and timeout > 0: + end_time += timeout + + while True: + # Try and create a hard link to it. + try: + os.link(self.unique_name, self.lock_file) + except OSError: + # Link creation failed. Maybe we've double-locked? + nlinks = os.stat(self.unique_name).st_nlink + if nlinks == 2: + # The original link plus the one I created == 2. We're + # good to go. + return + else: + # Otherwise the lock creation failed. + if timeout is not None and time.time() > end_time: + os.unlink(self.unique_name) + if timeout > 0: + raise LockTimeout("Timeout waiting to acquire" + " lock for %s" % + self.path) + else: + raise AlreadyLocked("%s is already locked" % + self.path) + time.sleep(timeout is not None and timeout/10 or 0.1) + else: + # Link creation succeeded. We're good to go. + return + + def release(self): + if not self.is_locked(): + raise NotLocked("%s is not locked" % self.path) + elif not os.path.exists(self.unique_name): + raise NotMyLock("%s is locked, but not by me" % self.path) + os.unlink(self.unique_name) + os.unlink(self.lock_file) + + def is_locked(self): + return os.path.exists(self.lock_file) + + def i_am_locking(self): + return (self.is_locked() and + os.path.exists(self.unique_name) and + os.stat(self.unique_name).st_nlink == 2) + + def break_lock(self): + if os.path.exists(self.lock_file): + os.unlink(self.lock_file) + diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile/mkdirlockfile.py b/scripts/external_libs/lockfile-0.10.2/lockfile/mkdirlockfile.py new file mode 100644 index 00000000..8d2c801f --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile/mkdirlockfile.py @@ -0,0 +1,83 @@ +from __future__ import absolute_import, division + +import time +import os +import sys +import errno + +from . import (LockBase, LockFailed, NotLocked, NotMyLock, LockTimeout, + AlreadyLocked) + +class MkdirLockFile(LockBase): + """Lock file by creating a directory.""" + def __init__(self, path, threaded=True, timeout=None): + """ + >>> lock = MkdirLockFile('somefile') + >>> lock = MkdirLockFile('somefile', threaded=False) + """ + LockBase.__init__(self, path, threaded, timeout) + # Lock file itself is a directory. Place the unique file name into + # it. + self.unique_name = os.path.join(self.lock_file, + "%s.%s%s" % (self.hostname, + self.tname, + self.pid)) + + def acquire(self, timeout=None): + timeout = timeout is not None and timeout or self.timeout + end_time = time.time() + if timeout is not None and timeout > 0: + end_time += timeout + + if timeout is None: + wait = 0.1 + else: + wait = max(0, timeout / 10) + + while True: + try: + os.mkdir(self.lock_file) + except OSError: + err = sys.exc_info()[1] + if err.errno == errno.EEXIST: + # Already locked. + if os.path.exists(self.unique_name): + # Already locked by me. + return + if timeout is not None and time.time() > end_time: + if timeout > 0: + raise LockTimeout("Timeout waiting to acquire" + " lock for %s" % + self.path) + else: + # Someone else has the lock. + raise AlreadyLocked("%s is already locked" % + self.path) + time.sleep(wait) + else: + # Couldn't create the lock for some other reason + raise LockFailed("failed to create %s" % self.lock_file) + else: + open(self.unique_name, "wb").close() + return + + def release(self): + if not self.is_locked(): + raise NotLocked("%s is not locked" % self.path) + elif not os.path.exists(self.unique_name): + raise NotMyLock("%s is locked, but not by me" % self.path) + os.unlink(self.unique_name) + os.rmdir(self.lock_file) + + def is_locked(self): + return os.path.exists(self.lock_file) + + def i_am_locking(self): + return (self.is_locked() and + os.path.exists(self.unique_name)) + + def break_lock(self): + if os.path.exists(self.lock_file): + for name in os.listdir(self.lock_file): + os.unlink(os.path.join(self.lock_file, name)) + os.rmdir(self.lock_file) diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile/pidlockfile.py b/scripts/external_libs/lockfile-0.10.2/lockfile/pidlockfile.py new file mode 100644 index 00000000..e92f9ead --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile/pidlockfile.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- + +# pidlockfile.py +# +# Copyright © 2008–2009 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Python Software Foundation License, version 2 or +# later as published by the Python Software Foundation. +# No warranty expressed or implied. See the file LICENSE.PSF-2 for details. + +""" Lockfile behaviour implemented via Unix PID files. + """ + +from __future__ import absolute_import + +import os +import sys +import errno +import time + +from . import (LockBase, AlreadyLocked, LockFailed, NotLocked, NotMyLock, + LockTimeout) + + +class PIDLockFile(LockBase): + """ Lockfile implemented as a Unix PID file. + + The lock file is a normal file named by the attribute `path`. + A lock's PID file contains a single line of text, containing + the process ID (PID) of the process that acquired the lock. + + >>> lock = PIDLockFile('somefile') + >>> lock = PIDLockFile('somefile') + """ + + def __init__(self, path, threaded=False, timeout=None): + # pid lockfiles don't support threaded operation, so always force + # False as the threaded arg. + LockBase.__init__(self, path, False, timeout) + dirname = os.path.dirname(self.lock_file) + basename = os.path.split(self.path)[-1] + self.unique_name = self.path + + def read_pid(self): + """ Get the PID from the lock file. + """ + return read_pid_from_pidfile(self.path) + + def is_locked(self): + """ Test if the lock is currently held. + + The lock is held if the PID file for this lock exists. + + """ + return os.path.exists(self.path) + + def i_am_locking(self): + """ Test if the lock is held by the current process. + + Returns ``True`` if the current process ID matches the + number stored in the PID file. + """ + return self.is_locked() and os.getpid() == self.read_pid() + + def acquire(self, timeout=None): + """ Acquire the lock. + + Creates the PID file for this lock, or raises an error if + the lock could not be acquired. + """ + + timeout = timeout is not None and timeout or self.timeout + end_time = time.time() + if timeout is not None and timeout > 0: + end_time += timeout + + while True: + try: + write_pid_to_pidfile(self.path) + except OSError as exc: + if exc.errno == errno.EEXIST: + # The lock creation failed. Maybe sleep a bit. + if timeout is not None and time.time() > end_time: + if timeout > 0: + raise LockTimeout("Timeout waiting to acquire" + " lock for %s" % + self.path) + else: + raise AlreadyLocked("%s is already locked" % + self.path) + time.sleep(timeout is not None and timeout/10 or 0.1) + else: + raise LockFailed("failed to create %s" % self.path) + else: + return + + def release(self): + """ Release the lock. + + Removes the PID file to release the lock, or raises an + error if the current process does not hold the lock. + + """ + if not self.is_locked(): + raise NotLocked("%s is not locked" % self.path) + if not self.i_am_locking(): + raise NotMyLock("%s is locked, but not by me" % self.path) + remove_existing_pidfile(self.path) + + def break_lock(self): + """ Break an existing lock. + + Removes the PID file if it already exists, otherwise does + nothing. + + """ + remove_existing_pidfile(self.path) + +def read_pid_from_pidfile(pidfile_path): + """ Read the PID recorded in the named PID file. + + Read and return the numeric PID recorded as text in the named + PID file. If the PID file cannot be read, or if the content is + not a valid PID, return ``None``. + + """ + pid = None + try: + pidfile = open(pidfile_path, 'r') + except IOError: + pass + else: + # According to the FHS 2.3 section on PID files in /var/run: + # + # The file must consist of the process identifier in + # ASCII-encoded decimal, followed by a newline character. + # + # Programs that read PID files should be somewhat flexible + # in what they accept; i.e., they should ignore extra + # whitespace, leading zeroes, absence of the trailing + # newline, or additional lines in the PID file. + + line = pidfile.readline().strip() + try: + pid = int(line) + except ValueError: + pass + pidfile.close() + + return pid + + +def write_pid_to_pidfile(pidfile_path): + """ Write the PID in the named PID file. + + Get the numeric process ID (“PID”) of the current process + and write it to the named file as a line of text. + + """ + open_flags = (os.O_CREAT | os.O_EXCL | os.O_WRONLY) + open_mode = 0o644 + pidfile_fd = os.open(pidfile_path, open_flags, open_mode) + pidfile = os.fdopen(pidfile_fd, 'w') + + # According to the FHS 2.3 section on PID files in /var/run: + # + # The file must consist of the process identifier in + # ASCII-encoded decimal, followed by a newline character. For + # example, if crond was process number 25, /var/run/crond.pid + # would contain three characters: two, five, and newline. + + pid = os.getpid() + line = "%(pid)d\n" % vars() + pidfile.write(line) + pidfile.close() + + +def remove_existing_pidfile(pidfile_path): + """ Remove the named PID file if it exists. + + Removing a PID file that doesn't already exist puts us in the + desired state, so we ignore the condition if the file does not + exist. + + """ + try: + os.remove(pidfile_path) + except OSError as exc: + if exc.errno == errno.ENOENT: + pass + else: + raise diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile/sqlitelockfile.py b/scripts/external_libs/lockfile-0.10.2/lockfile/sqlitelockfile.py new file mode 100644 index 00000000..7dee4a85 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile/sqlitelockfile.py @@ -0,0 +1,155 @@ +from __future__ import absolute_import, division + +import time +import os + +try: + unicode +except NameError: + unicode = str + +from . import LockBase, NotLocked, NotMyLock, LockTimeout, AlreadyLocked + +class SQLiteLockFile(LockBase): + "Demonstrate SQL-based locking." + + testdb = None + + def __init__(self, path, threaded=True, timeout=None): + """ + >>> lock = SQLiteLockFile('somefile') + >>> lock = SQLiteLockFile('somefile', threaded=False) + """ + LockBase.__init__(self, path, threaded, timeout) + self.lock_file = unicode(self.lock_file) + self.unique_name = unicode(self.unique_name) + + if SQLiteLockFile.testdb is None: + import tempfile + _fd, testdb = tempfile.mkstemp() + os.close(_fd) + os.unlink(testdb) + del _fd, tempfile + SQLiteLockFile.testdb = testdb + + import sqlite3 + self.connection = sqlite3.connect(SQLiteLockFile.testdb) + + c = self.connection.cursor() + try: + c.execute("create table locks" + "(" + " lock_file varchar(32)," + " unique_name varchar(32)" + ")") + except sqlite3.OperationalError: + pass + else: + self.connection.commit() + import atexit + atexit.register(os.unlink, SQLiteLockFile.testdb) + + def acquire(self, timeout=None): + timeout = timeout is not None and timeout or self.timeout + end_time = time.time() + if timeout is not None and timeout > 0: + end_time += timeout + + if timeout is None: + wait = 0.1 + elif timeout <= 0: + wait = 0 + else: + wait = timeout / 10 + + cursor = self.connection.cursor() + + while True: + if not self.is_locked(): + # Not locked. Try to lock it. + cursor.execute("insert into locks" + " (lock_file, unique_name)" + " values" + " (?, ?)", + (self.lock_file, self.unique_name)) + self.connection.commit() + + # Check to see if we are the only lock holder. + cursor.execute("select * from locks" + " where unique_name = ?", + (self.unique_name,)) + rows = cursor.fetchall() + if len(rows) > 1: + # Nope. Someone else got there. Remove our lock. + cursor.execute("delete from locks" + " where unique_name = ?", + (self.unique_name,)) + self.connection.commit() + else: + # Yup. We're done, so go home. + return + else: + # Check to see if we are the only lock holder. + cursor.execute("select * from locks" + " where unique_name = ?", + (self.unique_name,)) + rows = cursor.fetchall() + if len(rows) == 1: + # We're the locker, so go home. + return + + # Maybe we should wait a bit longer. + if timeout is not None and time.time() > end_time: + if timeout > 0: + # No more waiting. + raise LockTimeout("Timeout waiting to acquire" + " lock for %s" % + self.path) + else: + # Someone else has the lock and we are impatient.. + raise AlreadyLocked("%s is already locked" % self.path) + + # Well, okay. We'll give it a bit longer. + time.sleep(wait) + + def release(self): + if not self.is_locked(): + raise NotLocked("%s is not locked" % self.path) + if not self.i_am_locking(): + raise NotMyLock("%s is locked, but not by me (by %s)" % + (self.unique_name, self._who_is_locking())) + cursor = self.connection.cursor() + cursor.execute("delete from locks" + " where unique_name = ?", + (self.unique_name,)) + self.connection.commit() + + def _who_is_locking(self): + cursor = self.connection.cursor() + cursor.execute("select unique_name from locks" + " where lock_file = ?", + (self.lock_file,)) + return cursor.fetchone()[0] + + def is_locked(self): + cursor = self.connection.cursor() + cursor.execute("select * from locks" + " where lock_file = ?", + (self.lock_file,)) + rows = cursor.fetchall() + return not not rows + + def i_am_locking(self): + cursor = self.connection.cursor() + cursor.execute("select * from locks" + " where lock_file = ?" + " and unique_name = ?", + (self.lock_file, self.unique_name)) + return not not cursor.fetchall() + + def break_lock(self): + cursor = self.connection.cursor() + cursor.execute("delete from locks" + " where lock_file = ?", + (self.lock_file,)) + self.connection.commit() diff --git a/scripts/external_libs/lockfile-0.10.2/lockfile/symlinklockfile.py b/scripts/external_libs/lockfile-0.10.2/lockfile/symlinklockfile.py new file mode 100644 index 00000000..57551a36 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/lockfile/symlinklockfile.py @@ -0,0 +1,69 @@ +from __future__ import absolute_import + +import time +import os + +from . import (LockBase, LockFailed, NotLocked, NotMyLock, LockTimeout, + AlreadyLocked) + +class SymlinkLockFile(LockBase): + """Lock access to a file using symlink(2).""" + + def __init__(self, path, threaded=True, timeout=None): + # super(SymlinkLockFile).__init(...) + LockBase.__init__(self, path, threaded, timeout) + # split it back! + self.unique_name = os.path.split(self.unique_name)[1] + + def acquire(self, timeout=None): + # Hopefully unnecessary for symlink. + #try: + # open(self.unique_name, "wb").close() + #except IOError: + # raise LockFailed("failed to create %s" % self.unique_name) + timeout = timeout is not None and timeout or self.timeout + end_time = time.time() + if timeout is not None and timeout > 0: + end_time += timeout + + while True: + # Try and create a symbolic link to it. + try: + os.symlink(self.unique_name, self.lock_file) + except OSError: + # Link creation failed. Maybe we've double-locked? + if self.i_am_locking(): + # Linked to out unique name. Proceed. + return + else: + # Otherwise the lock creation failed. + if timeout is not None and time.time() > end_time: + if timeout > 0: + raise LockTimeout("Timeout waiting to acquire" + " lock for %s" % + self.path) + else: + raise AlreadyLocked("%s is already locked" % + self.path) + time.sleep(timeout/10 if timeout is not None else 0.1) + else: + # Link creation succeeded. We're good to go. + return + + def release(self): + if not self.is_locked(): + raise NotLocked("%s is not locked" % self.path) + elif not self.i_am_locking(): + raise NotMyLock("%s is locked, but not by me" % self.path) + os.unlink(self.lock_file) + + def is_locked(self): + return os.path.islink(self.lock_file) + + def i_am_locking(self): + return os.path.islink(self.lock_file) and \ + os.readlink(self.lock_file) == self.unique_name + + def break_lock(self): + if os.path.islink(self.lock_file): # exists && link + os.unlink(self.lock_file) diff --git a/scripts/external_libs/lockfile-0.10.2/setup.cfg b/scripts/external_libs/lockfile-0.10.2/setup.cfg new file mode 100644 index 00000000..c1fb3984 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/setup.cfg @@ -0,0 +1,39 @@ +[metadata] +name = lockfile +summary = Platform-independent file locking module +description-file = + README +author = OpenStack +author-email = openstack-dev@lists.openstack.org +home-page = http://launchpad.net/pylockfile +classifier = + Intended Audience :: Developers + License :: OSI Approved :: MIT License + Operating System :: POSIX :: Linux + Operating System :: MacOS + Operating System :: Microsoft :: Windows :: Windows NT/2000 + Operating System :: POSIX + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.3 + Topic :: Software Development :: Libraries :: Python Modules + +[files] +packages = lockfile + +[pbr] +warnerrors = true + +[build_sphinx] +source-dir = doc/source +build-dir = doc/build +all_files = 1 + +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 + diff --git a/scripts/external_libs/lockfile-0.10.2/setup.py b/scripts/external_libs/lockfile-0.10.2/setup.py new file mode 100644 index 00000000..73637574 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/setup.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/scripts/external_libs/lockfile-0.10.2/test-requirements.txt b/scripts/external_libs/lockfile-0.10.2/test-requirements.txt new file mode 100644 index 00000000..2e087ff1 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/test-requirements.txt @@ -0,0 +1,2 @@ +nose +sphinx>=1.1.2,!=1.2.0,<1.3 diff --git a/scripts/external_libs/lockfile-0.10.2/test/compliancetest.py b/scripts/external_libs/lockfile-0.10.2/test/compliancetest.py new file mode 100644 index 00000000..e0258b11 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/test/compliancetest.py @@ -0,0 +1,261 @@ +import os +import threading +import shutil + +import lockfile + +class ComplianceTest(object): + def __init__(self): + self.saved_class = lockfile.LockFile + + def _testfile(self): + """Return platform-appropriate file. Helper for tests.""" + import tempfile + return os.path.join(tempfile.gettempdir(), 'trash-%s' % os.getpid()) + + def setup(self): + lockfile.LockFile = self.class_to_test + + def teardown(self): + try: + tf = self._testfile() + if os.path.isdir(tf): + shutil.rmtree(tf) + elif os.path.isfile(tf): + os.unlink(tf) + elif not os.path.exists(tf): + pass + else: + raise SystemError("unrecognized file: %s" % tf) + finally: + lockfile.LockFile = self.saved_class + + def _test_acquire_helper(self, tbool): + # As simple as it gets. + lock = lockfile.LockFile(self._testfile(), threaded=tbool) + lock.acquire() + assert lock.i_am_locking() + lock.release() + assert not lock.is_locked() + +## def test_acquire_basic_threaded(self): +## self._test_acquire_helper(True) + + def test_acquire_basic_unthreaded(self): + self._test_acquire_helper(False) + + def _test_acquire_no_timeout_helper(self, tbool): + # No timeout test + e1, e2 = threading.Event(), threading.Event() + t = _in_thread(self._lock_wait_unlock, e1, e2) + e1.wait() # wait for thread t to acquire lock + lock2 = lockfile.LockFile(self._testfile(), threaded=tbool) + assert lock2.is_locked() + if tbool: + assert not lock2.i_am_locking() + else: + assert lock2.i_am_locking() + + try: + lock2.acquire(timeout=-1) + except lockfile.AlreadyLocked: + pass + else: + lock2.release() + raise AssertionError("did not raise AlreadyLocked in" + " thread %s" % + threading.current_thread().get_name()) + + try: + lock2.acquire(timeout=0) + except lockfile.AlreadyLocked: + pass + else: + lock2.release() + raise AssertionError("did not raise AlreadyLocked in" + " thread %s" % + threading.current_thread().get_name()) + + e2.set() # tell thread t to release lock + t.join() + +## def test_acquire_no_timeout_threaded(self): +## self._test_acquire_no_timeout_helper(True) + +## def test_acquire_no_timeout_unthreaded(self): +## self._test_acquire_no_timeout_helper(False) + + def _test_acquire_timeout_helper(self, tbool): + # Timeout test + e1, e2 = threading.Event(), threading.Event() + t = _in_thread(self._lock_wait_unlock, e1, e2) + e1.wait() # wait for thread t to acquire lock + lock2 = lockfile.LockFile(self._testfile(), threaded=tbool) + assert lock2.is_locked() + try: + lock2.acquire(timeout=0.1) + except lockfile.LockTimeout: + pass + else: + lock2.release() + raise AssertionError("did not raise LockTimeout in thread %s" % + threading.current_thread().get_name()) + + e2.set() + t.join() + + def test_acquire_timeout_threaded(self): + self._test_acquire_timeout_helper(True) + + def test_acquire_timeout_unthreaded(self): + self._test_acquire_timeout_helper(False) + + def _test_context_timeout_helper(self, tbool): + # Timeout test + e1, e2 = threading.Event(), threading.Event() + t = _in_thread(self._lock_wait_unlock, e1, e2) + e1.wait() # wait for thread t to acquire lock + lock2 = lockfile.LockFile(self._testfile(), threaded=tbool, + timeout=0.2) + assert lock2.is_locked() + try: + lock2.acquire() + except lockfile.LockTimeout: + pass + else: + lock2.release() + raise AssertionError("did not raise LockTimeout in thread %s" % + threading.current_thread().get_name()) + + e2.set() + t.join() + + def test_context_timeout_unthreaded(self): + self._test_context_timeout_helper(False) + + def _test_release_basic_helper(self, tbool): + lock = lockfile.LockFile(self._testfile(), threaded=tbool) + lock.acquire() + assert lock.is_locked() + lock.release() + assert not lock.is_locked() + assert not lock.i_am_locking() + try: + lock.release() + except lockfile.NotLocked: + pass + except lockfile.NotMyLock: + raise AssertionError('unexpected exception: %s' % + lockfile.NotMyLock) + else: + raise AssertionError('erroneously unlocked file') + +## def test_release_basic_threaded(self): +## self._test_release_basic_helper(True) + + def test_release_basic_unthreaded(self): + self._test_release_basic_helper(False) + +## def test_release_from_thread(self): +## e1, e2 = threading.Event(), threading.Event() +## t = _in_thread(self._lock_wait_unlock, e1, e2) +## e1.wait() +## lock2 = lockfile.LockFile(self._testfile(), threaded=False) +## assert not lock2.i_am_locking() +## try: +## lock2.release() +## except lockfile.NotMyLock: +## pass +## else: +## raise AssertionError('erroneously unlocked a file locked' +## ' by another thread.') +## e2.set() +## t.join() + + def _test_is_locked_helper(self, tbool): + lock = lockfile.LockFile(self._testfile(), threaded=tbool) + lock.acquire(timeout=2) + assert lock.is_locked() + lock.release() + assert not lock.is_locked(), "still locked after release!" + +## def test_is_locked_threaded(self): +## self._test_is_locked_helper(True) + + def test_is_locked_unthreaded(self): + self._test_is_locked_helper(False) + +## def test_i_am_locking_threaded(self): +## self._test_i_am_locking_helper(True) + + def test_i_am_locking_unthreaded(self): + self._test_i_am_locking_helper(False) + + def _test_i_am_locking_helper(self, tbool): + lock1 = lockfile.LockFile(self._testfile(), threaded=tbool) + assert not lock1.is_locked() + lock1.acquire() + try: + assert lock1.i_am_locking() + lock2 = lockfile.LockFile(self._testfile(), threaded=tbool) + assert lock2.is_locked() + if tbool: + assert not lock2.i_am_locking() + finally: + lock1.release() + + def _test_break_lock_helper(self, tbool): + lock = lockfile.LockFile(self._testfile(), threaded=tbool) + lock.acquire() + assert lock.is_locked() + lock2 = lockfile.LockFile(self._testfile(), threaded=tbool) + assert lock2.is_locked() + lock2.break_lock() + assert not lock2.is_locked() + try: + lock.release() + except lockfile.NotLocked: + pass + else: + raise AssertionError('break lock failed') + +## def test_break_lock_threaded(self): +## self._test_break_lock_helper(True) + + def test_break_lock_unthreaded(self): + self._test_break_lock_helper(False) + + def _lock_wait_unlock(self, event1, event2): + """Lock from another thread. Helper for tests.""" + l = lockfile.LockFile(self._testfile()) + l.acquire() + try: + event1.set() # we're in, + event2.wait() # wait for boss's permission to leave + finally: + l.release() + + def test_enter(self): + lock = lockfile.LockFile(self._testfile()) + lock.acquire() + try: + assert lock.is_locked(), "Not locked after acquire!" + finally: + lock.release() + assert not lock.is_locked(), "still locked after release!" + + def test_decorator(self): + @lockfile.locked(self._testfile()) + def func(a, b): + return a + b + assert func(4, 3) == 7 + +def _in_thread(func, *args, **kwargs): + """Execute func(*args, **kwargs) after dt seconds. Helper for tests.""" + def _f(): + func(*args, **kwargs) + t = threading.Thread(target=_f, name='/*/*') + t.setDaemon(True) + t.start() + return t + diff --git a/scripts/external_libs/lockfile-0.10.2/test/test_lockfile.py b/scripts/external_libs/lockfile-0.10.2/test/test_lockfile.py new file mode 100644 index 00000000..e1f4f72f --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/test/test_lockfile.py @@ -0,0 +1,36 @@ +import sys + +import lockfile.linklockfile +import lockfile.mkdirlockfile +import lockfile.pidlockfile +import lockfile.symlinklockfile + +from compliancetest import ComplianceTest + +class TestLinkLockFile(ComplianceTest): + class_to_test = lockfile.linklockfile.LinkLockFile + +class TestSymlinkLockFile(ComplianceTest): + class_to_test = lockfile.symlinklockfile.SymlinkLockFile + +class TestMkdirLockFile(ComplianceTest): + class_to_test = lockfile.mkdirlockfile.MkdirLockFile + +class TestPIDLockFile(ComplianceTest): + class_to_test = lockfile.pidlockfile.PIDLockFile + +# Check backwards compatibility +class TestLinkFileLock(ComplianceTest): + class_to_test = lockfile.LinkFileLock + +class TestMkdirFileLock(ComplianceTest): + class_to_test = lockfile.MkdirFileLock + +try: + import sqlite3 +except ImportError: + pass +else: + import lockfile.sqlitelockfile + class TestSQLiteLockFile(ComplianceTest): + class_to_test = lockfile.sqlitelockfile.SQLiteLockFile diff --git a/scripts/external_libs/lockfile-0.10.2/tox.ini b/scripts/external_libs/lockfile-0.10.2/tox.ini new file mode 100644 index 00000000..b0a868a3 --- /dev/null +++ b/scripts/external_libs/lockfile-0.10.2/tox.ini @@ -0,0 +1,28 @@ +# content of: tox.ini , put in same dir as setup.py +[tox] +envlist = py26,py27,py32,py33,py34 + +[testenv] +deps = -r{toxinidir}/test-requirements.txt +commands=nosetests + +[testenv:venv] +commands = {posargs} + +[testenv:pep8] +deps = flake8 +commands = flake8 + +[testenv:docs] +commands = python setup.py build_sphinx + +[testenv:cover] +deps = {[testenv]deps} + coverage +commands = + nosetests --with-coverage --cover-erase --cover-package=lockfile --cover-inclusive [] + +[flake8] +ignore = E121,E123,E128,E221,E226,E261,E265,E301,E302,E713,F401,F841,W291,W293,W391 +exclude=.venv,.git,.tox,dist,doc +show-source = True diff --git a/scripts/external_libs/python-daemon-2.0.5/ChangeLog b/scripts/external_libs/python-daemon-2.0.5/ChangeLog new file mode 100644 index 00000000..4975f781 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/ChangeLog @@ -0,0 +1,380 @@ +Version 2.0.5 +============= + +:Released: 2015-02-02 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Refine compatibility of exceptions for file operations. +* Specify the text encoding when opening the changelog file. + + +Version 2.0.4 +============= + +:Released: 2015-01-23 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Record version info via Setuptools commands. +* Remove the custom Setuptools entry points. + This closes Alioth bug#314948. + + +Version 2.0.3 +============= + +:Released: 2015-01-14 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Break circular import dependency for ‘setup.py’. +* Refactor all initial metadata functionality to ‘daemon._metadata’. +* Distribute ‘version’ (and its tests) only in source, not install. +* Build a “universal” (Python 2 and Python 3) wheel. + + +Version 2.0.2 +============= + +:Released: 2015-01-13 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Declare test-time dependency on recent ‘unittest2’. +* Declare packaging-time dependency on ‘docutils’ library. +* Include unit tests for ‘version’ module with source distribution. +* Record version info consistent with distribution metadata. + + +Version 2.0.1 +============= + +:Released: 2015-01-11 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Include the ‘version’ module with source distribution. + + +Version 2.0 +=========== + +:Released: 2015-01-10 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Support both Python 3 (version 3.2 or later) and Python 2 (version + 2.7 or later). +* Document the API of all functions comprehensively in docstrings. +* Add a hacking guide for developers. +* Add explicit credit for contributors. +* Document the security impact of the default umask. + +* Specify explicit text or binary mode when opening files. +* Preserve exception context in custom exceptions. + +* Declare compatibility with current Python versions. +* Depend on Python 3 compatible libraries. +* Update package homepage to Alioth hosted project page. +* Use ‘pydoc.splitdoc’ to get package description text. +* Remove ASCII translation of package description, not needed now the + docstring is a proper Unicode text value. +* Include test suite with source distribution. +* Move package metadata to ‘daemon/_metadata.py’. +* Migrate to JSON (instead of Python) for serialised version info. +* Add unit tests for metadata. +* Store and retrieve version info in Setuptools metadata. + +* Migrate to ‘str.format’ for interpolation of values into text. +* Migrate to ‘mock’ library for mock objects in tests. +* Migrate to ‘testscenarios’ library for unit test scenarios. +* Migrate to ‘unittest2’ library for back-ported improvements. + Remove custom test suite creation. +* Discriminate Python 2-and-3 compatible usage of dict methods. +* Discriminate Python 2-and-3 compatible bytes versus text. +* Declare explicit absolute and relative imports. +* Discriminate between different ‘fileno’ method behaviours. + In Python 3, ‘StringIO.fileno’ is callable but raises an exception. +* Migrate to built-in ‘next’ function. +* Wrap the ‘fromlist’ parameter of ‘__import__’ for Python 3 + compatibility. +* Wrap function introspection for Python 3 compatibility. +* Wrap standard library imports where names changed in Python 3. + + +Version 1.6.1 +============= + +:Released: 2014-08-04 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Use unambiguous “except FooType as foo” syntax. + This is to ease the port to Python 3, where the ambiguous comma + usage is an error. +* Ensure a ‘basestring’ name bound to the base type for strings. + This is to allow checks to work on Python 2 and 3. +* Specify versions of Python supported, as trove classifiers. + +* Update copyright notices. +* Add editor hints for most files. +* Distinguish continuation-line indentation versus block indentation. + +* Use unicode literals by default, specifying bytes where necessary. + This is to ease the port to Python 3, where the default string type + is unicode. +* Update copyright notices. +* Update the GPL license file to version 3, as declared in our + copyright notices. + +* Change license of library code to Apache License 2.0. Rationale at + <URL:http://wiki.python.org/moin/PythonSoftwareFoundationLicenseFaq#Contributing_Code_to_Python>. + + +Version 1.6 +=========== + +:Released: 2010-05-10 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Use absolute imports to disambiguate provenance of names. +* setup.py: Require ‘lockfile >=0.9’. +* daemon/pidfile.py: Renamed from ‘daemon/pidlockfile.py’. Change + references elsewhere to use this new name. +* test/test_pidfile.py: Renamed from ‘test/test_pidlockfile.py’. + Change references elsewhere to use this new name. +* daemon/pidfile.py: Remove functionality now migrated to ‘lockfile’ + library. + +* FAQ: Add some entries and re-structure the document. + +* Use ‘unicode’ data type for all text values. +* Prepare for Python 3 upgrade by tweaking some names and imports. + +* MANIFEST.in: Include the documentation in the distribution. + + +Version 1.5.5 +============= + +:Released: 2010-03-02 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Stop using ‘pkg_resources’ and revert to pre-1.5.3 version-string + handling, until a better way that doesn't break everyone else's + installation can be found. + + +Version 1.5.4 +============= + +:Released: 2010-02-27 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* MANIFEST.in: Explicitly include version data file, otherwise + everything breaks for users of the sdist. + + +Version 1.5.3 +============= + +:Released: 2010-02-26 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* daemon/daemon.py: Invoke the pidfile context manager's ‘__exit__’ + method with the correct arguments (as per + <URL:http://docs.python.org/library/stdtypes.html#typecontextmanager>). + Thanks to Ludvig Ericson for the bug report. +* version: New plain-text data file to store project version string. +* setup.py: Read version string from data file. +* daemon/version/__init__.py: Query version string with ‘pkg_resources’. + +* Add ‘pylint’ configuration for this project. +* Update copyright notices. + + +Version 1.5.2 +============= + +:Released: 2009-10-24 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Ensure we only prevent core dumps if ‘prevent_core’ is true. + Thanks to Denis Bilenko for reporting the lacking implementation of + this documented option. + +* Add initial Frequently Asked Questions document. + + +Version 1.5.1 +============= + +:Released: 2009-09-26 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Make a separate collection of DaemonRunner test scenarios. +* Handle a start request with a timeout on the PID file lock acquire. + +* Implement ‘TimeoutPIDLockFile’ to specify a timeout in advance of + lock acquisition. +* Use lock with timeout for ‘DaemonRunner’. + + +Version 1.5 +=========== + +:Released: 2009-09-24 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Make a separate collection of PIDLockFile test scenarios. + +* Raise specific errors on ‘DaemonRunner’ failures. +* Distinguish different conditions on reading and parsing PID file. +* Refactor code to ‘_terminate_daemon_process’ method. +* Improve explanations in comments and docstrings. +* Don't set pidfile at all if no path specified to constructor. +* Write the PID file using correct OS locking and permissions. +* Close the PID file after writing. +* Implement ‘PIDLockFile’ as subclass of ‘lockfile.LinkFileLock’. +* Remove redundant checks for file existence. + +* Manage the excluded file descriptors as a set (not a list). +* Only inspect the file descriptor of streams if they actually have + one (via a ‘fileno’ method) when determining which file descriptors + to close. Thanks to Ask Solem for revealing this bug. + + +Version 1.4.8 +============= + +:Released: 2009-09-17 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Remove child-exit signal (‘SIGCLD’, ‘SIGCHLD’) from default signal + map. Thanks to Joel Martin for pinpointing this issue. +* Document requirement for ensuring any operating-system specific + signal handlers are considered. +* Refactor ‘fork_then_exit_parent’ functionality to avoid duplicate + code. +* Remove redundant imports. +* Remove unused code from unit test suite scaffold. +* Add specific license terms for unit test suite scaffold. + + +Version 1.4.7 +============= + +:Released: 2009-09-03 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Fix keywords argument for distribution setup. +* Exclude ‘test’ package from distribution installation. + + +Version 1.4.6 +============= + +:Released: 2009-06-21 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Update documentation for changes from latest PEP 3143 revision. +* Implement DaemonContext.is_open method. + + +Version 1.4.5 +============= + +:Released: 2009-05-17 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Register DaemonContext.close method for atexit processing. +* Move PID file cleanup to close method. +* Improve docstrings by reference to, and copy from, PEP 3143. +* Use mock checking capabilities of newer ‘MiniMock’ library. +* Automate building a versioned distribution tarball. +* Include developer documentation files in source distribution. + + +Version 1.4.4 +============= + +:Released: 2009-03-26 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Conform to current PEP version, now released as PEP 3143 “Standard + daemon process library”. +* Ensure UID and GID are set in correct order. +* Delay closing all open files until just before re-binding standard + streams. +* Redirect standard streams to null device by default. + + +Version 1.4.3 +============= + +:Released: 2009-03-19 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Close the PID file context on exit. + + +Version 1.4.2 +============= + +:Released: 2009-03-18 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Context manager methods for DaemonContext. + + +Version 1.4.1 +============= + +:Released: 2009-03-18 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Improvements to docstrings. +* Further conformance with draft PEP. + + +Version 1.4 +=========== + +:Released: 2009-03-17 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Implement the interface from a draft PEP for process daemonisation. +* Complete statement coverage from unit test suite. + + +Version 1.3 +=========== + +:Released: 2009-03-12 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Separate controller (now ‘DaemonRunner’) from daemon process + context (now ‘DaemonContext’). +* Fix many corner cases and bugs. +* Huge increase in unit test suite. + + +Version 1.2 +=========== + +:Released: 2009-01-27 +:Maintainer: Ben Finney <ben+python@benfinney.id.au> + +* Initial release of this project forked from ‘bda.daemon’. Thanks, + Robert Niederreiter. +* Refactor some functionality out to helper functions. +* Begin unit test suite. + + +.. + This is free software: you may copy, modify, and/or distribute this work + under the terms of the Apache License version 2.0 as published by the + Apache Software Foundation. + No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +.. + Local variables: + coding: utf-8 + mode: text + mode: rst + End: + vim: fileencoding=utf-8 filetype=rst : diff --git a/scripts/external_libs/python-daemon-2.0.5/LICENSE.ASF-2 b/scripts/external_libs/python-daemon-2.0.5/LICENSE.ASF-2 new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/LICENSE.ASF-2 @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/scripts/external_libs/python-daemon-2.0.5/LICENSE.GPL-3 b/scripts/external_libs/python-daemon-2.0.5/LICENSE.GPL-3 new file mode 100644 index 00000000..94a9ed02 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/LICENSE.GPL-3 @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. diff --git a/scripts/external_libs/python-daemon-2.0.5/MANIFEST.in b/scripts/external_libs/python-daemon-2.0.5/MANIFEST.in new file mode 100644 index 00000000..d3d4341e --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/MANIFEST.in @@ -0,0 +1,7 @@ +include MANIFEST.in +include LICENSE.* +include ChangeLog +recursive-include doc * +include version.py +include test_version.py +recursive-include test *.py diff --git a/scripts/external_libs/python-daemon-2.0.5/PKG-INFO b/scripts/external_libs/python-daemon-2.0.5/PKG-INFO new file mode 100644 index 00000000..fd81f509 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/PKG-INFO @@ -0,0 +1,38 @@ +Metadata-Version: 1.1 +Name: python-daemon +Version: 2.0.5 +Summary: Library to implement a well-behaved Unix daemon process. +Home-page: https://alioth.debian.org/projects/python-daemon/ +Author: Ben Finney +Author-email: ben+python@benfinney.id.au +License: Apache-2 +Description: This library implements the well-behaved daemon specification of + :pep:`3143`, “Standard daemon process library”. + + A well-behaved Unix daemon process is tricky to get right, but the + required steps are much the same for every daemon program. A + `DaemonContext` instance holds the behaviour and configured + process environment for the program; use the instance as a context + manager to enter a daemon state. + + Simple example of usage:: + + import daemon + + from spam import do_main_program + + with daemon.DaemonContext(): + do_main_program() + + Customisation of the steps to become a daemon is available by + setting options on the `DaemonContext` instance; see the + documentation for that class for each option. +Keywords: daemon,fork,unix +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: POSIX +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Intended Audience :: Developers +Classifier: Topic :: Software Development :: Libraries :: Python Modules diff --git a/scripts/external_libs/python-daemon-2.0.5/daemon/__init__.py b/scripts/external_libs/python-daemon-2.0.5/daemon/__init__.py new file mode 100644 index 00000000..4731a6ef --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/daemon/__init__.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# daemon/__init__.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2009–2015 Ben Finney <ben+python@benfinney.id.au> +# Copyright © 2006 Robert Niederreiter +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Library to implement a well-behaved Unix daemon process. + + This library implements the well-behaved daemon specification of + :pep:`3143`, “Standard daemon process library”. + + A well-behaved Unix daemon process is tricky to get right, but the + required steps are much the same for every daemon program. A + `DaemonContext` instance holds the behaviour and configured + process environment for the program; use the instance as a context + manager to enter a daemon state. + + Simple example of usage:: + + import daemon + + from spam import do_main_program + + with daemon.DaemonContext(): + do_main_program() + + Customisation of the steps to become a daemon is available by + setting options on the `DaemonContext` instance; see the + documentation for that class for each option. + + """ + +from __future__ import (absolute_import, unicode_literals) + +from .daemon import DaemonContext + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/daemon/_metadata.py b/scripts/external_libs/python-daemon-2.0.5/daemon/_metadata.py new file mode 100644 index 00000000..6d22a2b7 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/daemon/_metadata.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +# daemon/_metadata.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Package metadata for the ‘python-daemon’ distribution. """ + +from __future__ import (absolute_import, unicode_literals) + +import json +import re +import collections +import datetime + +import pkg_resources + + +distribution_name = "python-daemon" +version_info_filename = "version_info.json" + +def get_distribution_version_info(filename=version_info_filename): + """ Get the version info from the installed distribution. + + :param filename: Base filename of the version info resource. + :return: The version info as a mapping of fields. If the + distribution is not available, the mapping is empty. + + The version info is stored as a metadata file in the + distribution. + + """ + version_info = { + 'release_date': "UNKNOWN", + 'version': "UNKNOWN", + 'maintainer': "UNKNOWN", + } + + try: + distribution = pkg_resources.get_distribution(distribution_name) + except pkg_resources.DistributionNotFound: + distribution = None + + if distribution is not None: + if distribution.has_metadata(version_info_filename): + content = distribution.get_metadata(version_info_filename) + version_info = json.loads(content) + + return version_info + +version_info = get_distribution_version_info() + +version_installed = version_info['version'] + + +rfc822_person_regex = re.compile( + "^(?P<name>[^<]+) <(?P<email>[^>]+)>$") + +ParsedPerson = collections.namedtuple('ParsedPerson', ['name', 'email']) + +def parse_person_field(value): + """ Parse a person field into name and email address. + + :param value: The text value specifying a person. + :return: A 2-tuple (name, email) for the person's details. + + If the `value` does not match a standard person with email + address, the `email` item is ``None``. + + """ + result = (None, None) + + match = rfc822_person_regex.match(value) + if len(value): + if match is not None: + result = ParsedPerson( + name=match.group('name'), + email=match.group('email')) + else: + result = ParsedPerson(name=value, email=None) + + return result + +author_name = "Ben Finney" +author_email = "ben+python@benfinney.id.au" +author = "{name} <{email}>".format(name=author_name, email=author_email) + + +class YearRange: + """ A range of years spanning a period. """ + + def __init__(self, begin, end=None): + self.begin = begin + self.end = end + + def __unicode__(self): + text = "{range.begin:04d}".format(range=self) + if self.end is not None: + if self.end > self.begin: + text = "{range.begin:04d}–{range.end:04d}".format(range=self) + return text + + __str__ = __unicode__ + + +def make_year_range(begin_year, end_date=None): + """ Construct the year range given a start and possible end date. + + :param begin_date: The beginning year (text) for the range. + :param end_date: The end date (text, ISO-8601 format) for the + range, or a non-date token string. + :return: The range of years as a `YearRange` instance. + + If the `end_date` is not a valid ISO-8601 date string, the + range has ``None`` for the end year. + + """ + begin_year = int(begin_year) + + try: + end_date = datetime.datetime.strptime(end_date, "%Y-%m-%d") + except (TypeError, ValueError): + # Specified end_date value is not a valid date. + end_year = None + else: + end_year = end_date.year + + year_range = YearRange(begin=begin_year, end=end_year) + + return year_range + +copyright_year_begin = "2001" +build_date = version_info['release_date'] +copyright_year_range = make_year_range(copyright_year_begin, build_date) + +copyright = "Copyright © {year_range} {author} and others".format( + year_range=copyright_year_range, author=author) +license = "Apache-2" +url = "https://alioth.debian.org/projects/python-daemon/" + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/daemon/daemon.py b/scripts/external_libs/python-daemon-2.0.5/daemon/daemon.py new file mode 100644 index 00000000..07810cf1 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/daemon/daemon.py @@ -0,0 +1,926 @@ +# -*- coding: utf-8 -*- + +# daemon/daemon.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au> +# Copyright © 2007–2008 Robert Niederreiter, Jens Klein +# Copyright © 2004–2005 Chad J. Schroeder +# Copyright © 2003 Clark Evans +# Copyright © 2002 Noah Spurrier +# Copyright © 2001 Jürgen Hermann +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Daemon process behaviour. + """ + +from __future__ import (absolute_import, unicode_literals) + +import os +import sys +import resource +import errno +import signal +import socket +import atexit +try: + # Python 2 has both ‘str’ (bytes) and ‘unicode’ (text). + basestring = basestring + unicode = unicode +except NameError: + # Python 3 names the Unicode data type ‘str’. + basestring = str + unicode = str + + +class DaemonError(Exception): + """ Base exception class for errors from this module. """ + + def __init__(self, *args, **kwargs): + self._chain_from_context() + + super(DaemonError, self).__init__(*args, **kwargs) + + def _chain_from_context(self): + _chain_exception_from_existing_exception_context(self, as_cause=True) + + +class DaemonOSEnvironmentError(DaemonError, OSError): + """ Exception raised when daemon OS environment setup receives error. """ + + +class DaemonProcessDetachError(DaemonError, OSError): + """ Exception raised when process detach fails. """ + + +class DaemonContext: + """ Context for turning the current program into a daemon process. + + A `DaemonContext` instance represents the behaviour settings and + process context for the program when it becomes a daemon. The + behaviour and environment is customised by setting options on the + instance, before calling the `open` method. + + Each option can be passed as a keyword argument to the `DaemonContext` + constructor, or subsequently altered by assigning to an attribute on + the instance at any time prior to calling `open`. That is, for + options named `wibble` and `wubble`, the following invocation:: + + foo = daemon.DaemonContext(wibble=bar, wubble=baz) + foo.open() + + is equivalent to:: + + foo = daemon.DaemonContext() + foo.wibble = bar + foo.wubble = baz + foo.open() + + The following options are defined. + + `files_preserve` + :Default: ``None`` + + List of files that should *not* be closed when starting the + daemon. If ``None``, all open file descriptors will be closed. + + Elements of the list are file descriptors (as returned by a file + object's `fileno()` method) or Python `file` objects. Each + specifies a file that is not to be closed during daemon start. + + `chroot_directory` + :Default: ``None`` + + Full path to a directory to set as the effective root directory of + the process. If ``None``, specifies that the root directory is not + to be changed. + + `working_directory` + :Default: ``'/'`` + + Full path of the working directory to which the process should + change on daemon start. + + Since a filesystem cannot be unmounted if a process has its + current working directory on that filesystem, this should either + be left at default or set to a directory that is a sensible “home + directory” for the daemon while it is running. + + `umask` + :Default: ``0`` + + File access creation mask (“umask”) to set for the process on + daemon start. + + A daemon should not rely on the parent process's umask value, + which is beyond its control and may prevent creating a file with + the required access mode. So when the daemon context opens, the + umask is set to an explicit known value. + + If the conventional value of 0 is too open, consider setting a + value such as 0o022, 0o027, 0o077, or another specific value. + Otherwise, ensure the daemon creates every file with an + explicit access mode for the purpose. + + `pidfile` + :Default: ``None`` + + Context manager for a PID lock file. When the daemon context opens + and closes, it enters and exits the `pidfile` context manager. + + `detach_process` + :Default: ``None`` + + If ``True``, detach the process context when opening the daemon + context; if ``False``, do not detach. + + If unspecified (``None``) during initialisation of the instance, + this will be set to ``True`` by default, and ``False`` only if + detaching the process is determined to be redundant; for example, + in the case when the process was started by `init`, by `initd`, or + by `inetd`. + + `signal_map` + :Default: system-dependent + + Mapping from operating system signals to callback actions. + + The mapping is used when the daemon context opens, and determines + the action for each signal's signal handler: + + * A value of ``None`` will ignore the signal (by setting the + signal action to ``signal.SIG_IGN``). + + * A string value will be used as the name of an attribute on the + ``DaemonContext`` instance. The attribute's value will be used + as the action for the signal handler. + + * Any other value will be used as the action for the + signal handler. See the ``signal.signal`` documentation + for details of the signal handler interface. + + The default value depends on which signals are defined on the + running system. Each item from the list below whose signal is + actually defined in the ``signal`` module will appear in the + default map: + + * ``signal.SIGTTIN``: ``None`` + + * ``signal.SIGTTOU``: ``None`` + + * ``signal.SIGTSTP``: ``None`` + + * ``signal.SIGTERM``: ``'terminate'`` + + Depending on how the program will interact with its child + processes, it may need to specify a signal map that + includes the ``signal.SIGCHLD`` signal (received when a + child process exits). See the specific operating system's + documentation for more detail on how to determine what + circumstances dictate the need for signal handlers. + + `uid` + :Default: ``os.getuid()`` + + `gid` + :Default: ``os.getgid()`` + + The user ID (“UID”) value and group ID (“GID”) value to switch + the process to on daemon start. + + The default values, the real UID and GID of the process, will + relinquish any effective privilege elevation inherited by the + process. + + `prevent_core` + :Default: ``True`` + + If true, prevents the generation of core files, in order to avoid + leaking sensitive information from daemons run as `root`. + + `stdin` + :Default: ``None`` + + `stdout` + :Default: ``None`` + + `stderr` + :Default: ``None`` + + Each of `stdin`, `stdout`, and `stderr` is a file-like object + which will be used as the new file for the standard I/O stream + `sys.stdin`, `sys.stdout`, and `sys.stderr` respectively. The file + should therefore be open, with a minimum of mode 'r' in the case + of `stdin`, and mimimum of mode 'w+' in the case of `stdout` and + `stderr`. + + If the object has a `fileno()` method that returns a file + descriptor, the corresponding file will be excluded from being + closed during daemon start (that is, it will be treated as though + it were listed in `files_preserve`). + + If ``None``, the corresponding system stream is re-bound to the + file named by `os.devnull`. + + """ + + __metaclass__ = type + + def __init__( + self, + chroot_directory=None, + working_directory="/", + umask=0, + uid=None, + gid=None, + prevent_core=True, + detach_process=None, + files_preserve=None, + pidfile=None, + stdin=None, + stdout=None, + stderr=None, + signal_map=None, + ): + """ Set up a new instance. """ + self.chroot_directory = chroot_directory + self.working_directory = working_directory + self.umask = umask + self.prevent_core = prevent_core + self.files_preserve = files_preserve + self.pidfile = pidfile + self.stdin = stdin + self.stdout = stdout + self.stderr = stderr + + if uid is None: + uid = os.getuid() + self.uid = uid + if gid is None: + gid = os.getgid() + self.gid = gid + + if detach_process is None: + detach_process = is_detach_process_context_required() + self.detach_process = detach_process + + if signal_map is None: + signal_map = make_default_signal_map() + self.signal_map = signal_map + + self._is_open = False + + @property + def is_open(self): + """ ``True`` if the instance is currently open. """ + return self._is_open + + def open(self): + """ Become a daemon process. + + :return: ``None``. + + Open the daemon context, turning the current program into a daemon + process. This performs the following steps: + + * If this instance's `is_open` property is true, return + immediately. This makes it safe to call `open` multiple times on + an instance. + + * If the `prevent_core` attribute is true, set the resource limits + for the process to prevent any core dump from the process. + + * If the `chroot_directory` attribute is not ``None``, set the + effective root directory of the process to that directory (via + `os.chroot`). + + This allows running the daemon process inside a “chroot gaol” + as a means of limiting the system's exposure to rogue behaviour + by the process. Note that the specified directory needs to + already be set up for this purpose. + + * Set the process UID and GID to the `uid` and `gid` attribute + values. + + * Close all open file descriptors. This excludes those listed in + the `files_preserve` attribute, and those that correspond to the + `stdin`, `stdout`, or `stderr` attributes. + + * Change current working directory to the path specified by the + `working_directory` attribute. + + * Reset the file access creation mask to the value specified by + the `umask` attribute. + + * If the `detach_process` option is true, detach the current + process into its own process group, and disassociate from any + controlling terminal. + + * Set signal handlers as specified by the `signal_map` attribute. + + * If any of the attributes `stdin`, `stdout`, `stderr` are not + ``None``, bind the system streams `sys.stdin`, `sys.stdout`, + and/or `sys.stderr` to the files represented by the + corresponding attributes. Where the attribute has a file + descriptor, the descriptor is duplicated (instead of re-binding + the name). + + * If the `pidfile` attribute is not ``None``, enter its context + manager. + + * Mark this instance as open (for the purpose of future `open` and + `close` calls). + + * Register the `close` method to be called during Python's exit + processing. + + When the function returns, the running program is a daemon + process. + + """ + if self.is_open: + return + + if self.chroot_directory is not None: + change_root_directory(self.chroot_directory) + + if self.prevent_core: + prevent_core_dump() + + change_file_creation_mask(self.umask) + change_working_directory(self.working_directory) + change_process_owner(self.uid, self.gid) + + if self.detach_process: + detach_process_context() + + signal_handler_map = self._make_signal_handler_map() + set_signal_handlers(signal_handler_map) + + exclude_fds = self._get_exclude_file_descriptors() + close_all_open_files(exclude=exclude_fds) + + redirect_stream(sys.stdin, self.stdin) + redirect_stream(sys.stdout, self.stdout) + redirect_stream(sys.stderr, self.stderr) + + if self.pidfile is not None: + self.pidfile.__enter__() + + self._is_open = True + + register_atexit_function(self.close) + + def __enter__(self): + """ Context manager entry point. """ + self.open() + return self + + def close(self): + """ Exit the daemon process context. + + :return: ``None``. + + Close the daemon context. This performs the following steps: + + * If this instance's `is_open` property is false, return + immediately. This makes it safe to call `close` multiple times + on an instance. + + * If the `pidfile` attribute is not ``None``, exit its context + manager. + + * Mark this instance as closed (for the purpose of future `open` + and `close` calls). + + """ + if not self.is_open: + return + + if self.pidfile is not None: + # Follow the interface for telling a context manager to exit, + # <URL:http://docs.python.org/library/stdtypes.html#typecontextmanager>. + self.pidfile.__exit__(None, None, None) + + self._is_open = False + + def __exit__(self, exc_type, exc_value, traceback): + """ Context manager exit point. """ + self.close() + + def terminate(self, signal_number, stack_frame): + """ Signal handler for end-process signals. + + :param signal_number: The OS signal number received. + :param stack_frame: The frame object at the point the + signal was received. + :return: ``None``. + + Signal handler for the ``signal.SIGTERM`` signal. Performs the + following step: + + * Raise a ``SystemExit`` exception explaining the signal. + + """ + exception = SystemExit( + "Terminating on signal {signal_number!r}".format( + signal_number=signal_number)) + raise exception + + def _get_exclude_file_descriptors(self): + """ Get the set of file descriptors to exclude closing. + + :return: A set containing the file descriptors for the + files to be preserved. + + The file descriptors to be preserved are those from the + items in `files_preserve`, and also each of `stdin`, + `stdout`, and `stderr`. For each item: + + * If the item is ``None``, it is omitted from the return + set. + + * If the item's ``fileno()`` method returns a value, that + value is in the return set. + + * Otherwise, the item is in the return set verbatim. + + """ + files_preserve = self.files_preserve + if files_preserve is None: + files_preserve = [] + files_preserve.extend( + item for item in [self.stdin, self.stdout, self.stderr] + if hasattr(item, 'fileno')) + + exclude_descriptors = set() + for item in files_preserve: + if item is None: + continue + file_descriptor = _get_file_descriptor(item) + if file_descriptor is not None: + exclude_descriptors.add(file_descriptor) + else: + exclude_descriptors.add(item) + + return exclude_descriptors + + def _make_signal_handler(self, target): + """ Make the signal handler for a specified target object. + + :param target: A specification of the target for the + handler; see below. + :return: The value for use by `signal.signal()`. + + If `target` is ``None``, return ``signal.SIG_IGN``. If `target` + is a text string, return the attribute of this instance named + by that string. Otherwise, return `target` itself. + + """ + if target is None: + result = signal.SIG_IGN + elif isinstance(target, unicode): + name = target + result = getattr(self, name) + else: + result = target + + return result + + def _make_signal_handler_map(self): + """ Make the map from signals to handlers for this instance. + + :return: The constructed signal map for this instance. + + Construct a map from signal numbers to handlers for this + context instance, suitable for passing to + `set_signal_handlers`. + + """ + signal_handler_map = dict( + (signal_number, self._make_signal_handler(target)) + for (signal_number, target) in self.signal_map.items()) + return signal_handler_map + + +def _get_file_descriptor(obj): + """ Get the file descriptor, if the object has one. + + :param obj: The object expected to be a file-like object. + :return: The file descriptor iff the file supports it; otherwise + ``None``. + + The object may be a non-file object. It may also be a + file-like object with no support for a file descriptor. In + either case, return ``None``. + + """ + file_descriptor = None + if hasattr(obj, 'fileno'): + try: + file_descriptor = obj.fileno() + except ValueError: + # The item doesn't support a file descriptor. + pass + + return file_descriptor + + +def change_working_directory(directory): + """ Change the working directory of this process. + + :param directory: The target directory path. + :return: ``None``. + + """ + try: + os.chdir(directory) + except Exception as exc: + error = DaemonOSEnvironmentError( + "Unable to change working directory ({exc})".format(exc=exc)) + raise error + + +def change_root_directory(directory): + """ Change the root directory of this process. + + :param directory: The target directory path. + :return: ``None``. + + Set the current working directory, then the process root directory, + to the specified `directory`. Requires appropriate OS privileges + for this process. + + """ + try: + os.chdir(directory) + os.chroot(directory) + except Exception as exc: + error = DaemonOSEnvironmentError( + "Unable to change root directory ({exc})".format(exc=exc)) + raise error + + +def change_file_creation_mask(mask): + """ Change the file creation mask for this process. + + :param mask: The numeric file creation mask to set. + :return: ``None``. + + """ + try: + os.umask(mask) + except Exception as exc: + error = DaemonOSEnvironmentError( + "Unable to change file creation mask ({exc})".format(exc=exc)) + raise error + + +def change_process_owner(uid, gid): + """ Change the owning UID and GID of this process. + + :param uid: The target UID for the daemon process. + :param gid: The target GID for the daemon process. + :return: ``None``. + + Set the GID then the UID of the process (in that order, to avoid + permission errors) to the specified `gid` and `uid` values. + Requires appropriate OS privileges for this process. + + """ + try: + os.setgid(gid) + os.setuid(uid) + except Exception as exc: + error = DaemonOSEnvironmentError( + "Unable to change process owner ({exc})".format(exc=exc)) + raise error + + +def prevent_core_dump(): + """ Prevent this process from generating a core dump. + + :return: ``None``. + + Set the soft and hard limits for core dump size to zero. On Unix, + this entirely prevents the process from creating core dump. + + """ + core_resource = resource.RLIMIT_CORE + + try: + # Ensure the resource limit exists on this platform, by requesting + # its current value. + core_limit_prev = resource.getrlimit(core_resource) + except ValueError as exc: + error = DaemonOSEnvironmentError( + "System does not support RLIMIT_CORE resource limit" + " ({exc})".format(exc=exc)) + raise error + + # Set hard and soft limits to zero, i.e. no core dump at all. + core_limit = (0, 0) + resource.setrlimit(core_resource, core_limit) + + +def detach_process_context(): + """ Detach the process context from parent and session. + + :return: ``None``. + + Detach from the parent process and session group, allowing the + parent to exit while this process continues running. + + Reference: “Advanced Programming in the Unix Environment”, + section 13.3, by W. Richard Stevens, published 1993 by + Addison-Wesley. + + """ + + def fork_then_exit_parent(error_message): + """ Fork a child process, then exit the parent process. + + :param error_message: Message for the exception in case of a + detach failure. + :return: ``None``. + :raise DaemonProcessDetachError: If the fork fails. + + """ + try: + pid = os.fork() + if pid > 0: + os._exit(0) + except OSError as exc: + error = DaemonProcessDetachError( + "{message}: [{exc.errno:d}] {exc.strerror}".format( + message=error_message, exc=exc)) + raise error + + fork_then_exit_parent(error_message="Failed first fork") + os.setsid() + fork_then_exit_parent(error_message="Failed second fork") + + +def is_process_started_by_init(): + """ Determine whether the current process is started by `init`. + + :return: ``True`` iff the parent process is `init`; otherwise + ``False``. + + The `init` process is the one with process ID of 1. + + """ + result = False + + init_pid = 1 + if os.getppid() == init_pid: + result = True + + return result + + +def is_socket(fd): + """ Determine whether the file descriptor is a socket. + + :param fd: The file descriptor to interrogate. + :return: ``True`` iff the file descriptor is a socket; otherwise + ``False``. + + Query the socket type of `fd`. If there is no error, the file is a + socket. + + """ + result = False + + file_socket = socket.fromfd(fd, socket.AF_INET, socket.SOCK_RAW) + + try: + socket_type = file_socket.getsockopt( + socket.SOL_SOCKET, socket.SO_TYPE) + except socket.error as exc: + exc_errno = exc.args[0] + if exc_errno == errno.ENOTSOCK: + # Socket operation on non-socket. + pass + else: + # Some other socket error. + result = True + else: + # No error getting socket type. + result = True + + return result + + +def is_process_started_by_superserver(): + """ Determine whether the current process is started by the superserver. + + :return: ``True`` if this process was started by the internet + superserver; otherwise ``False``. + + The internet superserver creates a network socket, and + attaches it to the standard streams of the child process. If + that is the case for this process, return ``True``, otherwise + ``False``. + + """ + result = False + + stdin_fd = sys.__stdin__.fileno() + if is_socket(stdin_fd): + result = True + + return result + + +def is_detach_process_context_required(): + """ Determine whether detaching the process context is required. + + :return: ``True`` iff the process is already detached; otherwise + ``False``. + + The process environment is interrogated for the following: + + * Process was started by `init`; or + + * Process was started by `inetd`. + + If any of the above are true, the process is deemed to be already + detached. + + """ + result = True + if is_process_started_by_init() or is_process_started_by_superserver(): + result = False + + return result + + +def close_file_descriptor_if_open(fd): + """ Close a file descriptor if already open. + + :param fd: The file descriptor to close. + :return: ``None``. + + Close the file descriptor `fd`, suppressing an error in the + case the file was not open. + + """ + try: + os.close(fd) + except EnvironmentError as exc: + if exc.errno == errno.EBADF: + # File descriptor was not open. + pass + else: + error = DaemonOSEnvironmentError( + "Failed to close file descriptor {fd:d} ({exc})".format( + fd=fd, exc=exc)) + raise error + + +MAXFD = 2048 + +def get_maximum_file_descriptors(): + """ Get the maximum number of open file descriptors for this process. + + :return: The number (integer) to use as the maximum number of open + files for this process. + + The maximum is the process hard resource limit of maximum number of + open file descriptors. If the limit is “infinity”, a default value + of ``MAXFD`` is returned. + + """ + limits = resource.getrlimit(resource.RLIMIT_NOFILE) + result = limits[1] + if result == resource.RLIM_INFINITY: + result = MAXFD + return result + + +def close_all_open_files(exclude=set()): + """ Close all open file descriptors. + + :param exclude: Collection of file descriptors to skip when closing + files. + :return: ``None``. + + Closes every file descriptor (if open) of this process. If + specified, `exclude` is a set of file descriptors to *not* + close. + + """ + maxfd = get_maximum_file_descriptors() + for fd in reversed(range(maxfd)): + if fd not in exclude: + close_file_descriptor_if_open(fd) + + +def redirect_stream(system_stream, target_stream): + """ Redirect a system stream to a specified file. + + :param standard_stream: A file object representing a standard I/O + stream. + :param target_stream: The target file object for the redirected + stream, or ``None`` to specify the null device. + :return: ``None``. + + `system_stream` is a standard system stream such as + ``sys.stdout``. `target_stream` is an open file object that + should replace the corresponding system stream object. + + If `target_stream` is ``None``, defaults to opening the + operating system's null device and using its file descriptor. + + """ + if target_stream is None: + target_fd = os.open(os.devnull, os.O_RDWR) + else: + target_fd = target_stream.fileno() + os.dup2(target_fd, system_stream.fileno()) + + +def make_default_signal_map(): + """ Make the default signal map for this system. + + :return: A mapping from signal number to handler object. + + The signals available differ by system. The map will not contain + any signals not defined on the running system. + + """ + name_map = { + 'SIGTSTP': None, + 'SIGTTIN': None, + 'SIGTTOU': None, + 'SIGTERM': 'terminate', + } + signal_map = dict( + (getattr(signal, name), target) + for (name, target) in name_map.items() + if hasattr(signal, name)) + + return signal_map + + +def set_signal_handlers(signal_handler_map): + """ Set the signal handlers as specified. + + :param signal_handler_map: A map from signal number to handler + object. + :return: ``None``. + + See the `signal` module for details on signal numbers and signal + handlers. + + """ + for (signal_number, handler) in signal_handler_map.items(): + signal.signal(signal_number, handler) + + +def register_atexit_function(func): + """ Register a function for processing at program exit. + + :param func: A callable function expecting no arguments. + :return: ``None``. + + The function `func` is registered for a call with no arguments + at program exit. + + """ + atexit.register(func) + + +def _chain_exception_from_existing_exception_context(exc, as_cause=False): + """ Decorate the specified exception with the existing exception context. + + :param exc: The exception instance to decorate. + :param as_cause: If true, the existing context is declared to be + the cause of the exception. + :return: ``None``. + + :PEP:`344` describes syntax and attributes (`__traceback__`, + `__context__`, `__cause__`) for use in exception chaining. + + Python 2 does not have that syntax, so this function decorates + the exception with values from the current exception context. + + """ + (existing_exc_type, existing_exc, existing_traceback) = sys.exc_info() + if as_cause: + exc.__cause__ = existing_exc + else: + exc.__context__ = existing_exc + exc.__traceback__ = existing_traceback + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/daemon/pidfile.py b/scripts/external_libs/python-daemon-2.0.5/daemon/pidfile.py new file mode 100644 index 00000000..4517ee0e --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/daemon/pidfile.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +# daemon/pidfile.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Lockfile behaviour implemented via Unix PID files. + """ + +from __future__ import (absolute_import, unicode_literals) + +from lockfile.pidlockfile import PIDLockFile + + +class TimeoutPIDLockFile(PIDLockFile, object): + """ Lockfile with default timeout, implemented as a Unix PID file. + + This uses the ``PIDLockFile`` implementation, with the + following changes: + + * The `acquire_timeout` parameter to the initialiser will be + used as the default `timeout` parameter for the `acquire` + method. + + """ + + def __init__(self, path, acquire_timeout=None, *args, **kwargs): + """ Set up the parameters of a TimeoutPIDLockFile. + + :param path: Filesystem path to the PID file. + :param acquire_timeout: Value to use by default for the + `acquire` call. + :return: ``None``. + + """ + self.acquire_timeout = acquire_timeout + super(TimeoutPIDLockFile, self).__init__(path, *args, **kwargs) + + def acquire(self, timeout=None, *args, **kwargs): + """ Acquire the lock. + + :param timeout: Specifies the timeout; see below for valid + values. + :return: ``None``. + + The `timeout` defaults to the value set during + initialisation with the `acquire_timeout` parameter. It is + passed to `PIDLockFile.acquire`; see that method for + details. + + """ + if timeout is None: + timeout = self.acquire_timeout + super(TimeoutPIDLockFile, self).acquire(timeout, *args, **kwargs) + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/daemon/runner.py b/scripts/external_libs/python-daemon-2.0.5/daemon/runner.py new file mode 100644 index 00000000..6973cf1c --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/daemon/runner.py @@ -0,0 +1,324 @@ +# -*- coding: utf-8 -*- + +# daemon/runner.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2009–2015 Ben Finney <ben+python@benfinney.id.au> +# Copyright © 2007–2008 Robert Niederreiter, Jens Klein +# Copyright © 2003 Clark Evans +# Copyright © 2002 Noah Spurrier +# Copyright © 2001 Jürgen Hermann +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Daemon runner library. + """ + +from __future__ import (absolute_import, unicode_literals) + +import sys +import os +import signal +import errno +try: + # Python 3 standard library. + ProcessLookupError +except NameError: + # No such class in Python 2. + ProcessLookupError = NotImplemented + +import lockfile + +from . import pidfile +from .daemon import (basestring, unicode) +from .daemon import DaemonContext +from .daemon import _chain_exception_from_existing_exception_context + + +class DaemonRunnerError(Exception): + """ Abstract base class for errors from DaemonRunner. """ + + def __init__(self, *args, **kwargs): + self._chain_from_context() + + super(DaemonRunnerError, self).__init__(*args, **kwargs) + + def _chain_from_context(self): + _chain_exception_from_existing_exception_context(self, as_cause=True) + + +class DaemonRunnerInvalidActionError(DaemonRunnerError, ValueError): + """ Raised when specified action for DaemonRunner is invalid. """ + + def _chain_from_context(self): + # This exception is normally not caused by another. + _chain_exception_from_existing_exception_context(self, as_cause=False) + + +class DaemonRunnerStartFailureError(DaemonRunnerError, RuntimeError): + """ Raised when failure starting DaemonRunner. """ + + +class DaemonRunnerStopFailureError(DaemonRunnerError, RuntimeError): + """ Raised when failure stopping DaemonRunner. """ + + +class DaemonRunner: + """ Controller for a callable running in a separate background process. + + The first command-line argument is the action to take: + + * 'start': Become a daemon and call `app.run()`. + * 'stop': Exit the daemon process specified in the PID file. + * 'restart': Stop, then start. + + """ + + __metaclass__ = type + + start_message = "started with pid {pid:d}" + + def __init__(self, app): + """ Set up the parameters of a new runner. + + :param app: The application instance; see below. + :return: ``None``. + + The `app` argument must have the following attributes: + + * `stdin_path`, `stdout_path`, `stderr_path`: Filesystem paths + to open and replace the existing `sys.stdin`, `sys.stdout`, + `sys.stderr`. + + * `pidfile_path`: Absolute filesystem path to a file that will + be used as the PID file for the daemon. If ``None``, no PID + file will be used. + + * `pidfile_timeout`: Used as the default acquisition timeout + value supplied to the runner's PID lock file. + + * `run`: Callable that will be invoked when the daemon is + started. + + """ + self.parse_args() + self.app = app + self.daemon_context = DaemonContext() + self.daemon_context.stdin = open(app.stdin_path, 'rt') + self.daemon_context.stdout = open(app.stdout_path, 'w+t') + self.daemon_context.stderr = open( + app.stderr_path, 'w+t', buffering=0) + + self.pidfile = None + if app.pidfile_path is not None: + self.pidfile = make_pidlockfile( + app.pidfile_path, app.pidfile_timeout) + self.daemon_context.pidfile = self.pidfile + + def _usage_exit(self, argv): + """ Emit a usage message, then exit. + + :param argv: The command-line arguments used to invoke the + program, as a sequence of strings. + :return: ``None``. + + """ + progname = os.path.basename(argv[0]) + usage_exit_code = 2 + action_usage = "|".join(self.action_funcs.keys()) + message = "usage: {progname} {usage}".format( + progname=progname, usage=action_usage) + emit_message(message) + sys.exit(usage_exit_code) + + def parse_args(self, argv=None): + """ Parse command-line arguments. + + :param argv: The command-line arguments used to invoke the + program, as a sequence of strings. + + :return: ``None``. + + The parser expects the first argument as the program name, the + second argument as the action to perform. + + If the parser fails to parse the arguments, emit a usage + message and exit the program. + + """ + if argv is None: + argv = sys.argv + + min_args = 2 + if len(argv) < min_args: + self._usage_exit(argv) + + self.action = unicode(argv[1]) + if self.action not in self.action_funcs: + self._usage_exit(argv) + + def _start(self): + """ Open the daemon context and run the application. + + :return: ``None``. + :raises DaemonRunnerStartFailureError: If the PID file cannot + be locked by this process. + + """ + if is_pidfile_stale(self.pidfile): + self.pidfile.break_lock() + + try: + self.daemon_context.open() + except lockfile.AlreadyLocked: + error = DaemonRunnerStartFailureError( + "PID file {pidfile.path!r} already locked".format( + pidfile=self.pidfile)) + raise error + + pid = os.getpid() + message = self.start_message.format(pid=pid) + emit_message(message) + + self.app.run() + + def _terminate_daemon_process(self): + """ Terminate the daemon process specified in the current PID file. + + :return: ``None``. + :raises DaemonRunnerStopFailureError: If terminating the daemon + fails with an OS error. + + """ + pid = self.pidfile.read_pid() + try: + os.kill(pid, signal.SIGTERM) + except OSError as exc: + error = DaemonRunnerStopFailureError( + "Failed to terminate {pid:d}: {exc}".format( + pid=pid, exc=exc)) + raise error + + def _stop(self): + """ Exit the daemon process specified in the current PID file. + + :return: ``None``. + :raises DaemonRunnerStopFailureError: If the PID file is not + already locked. + + """ + if not self.pidfile.is_locked(): + error = DaemonRunnerStopFailureError( + "PID file {pidfile.path!r} not locked".format( + pidfile=self.pidfile)) + raise error + + if is_pidfile_stale(self.pidfile): + self.pidfile.break_lock() + else: + self._terminate_daemon_process() + + def _restart(self): + """ Stop, then start. + """ + self._stop() + self._start() + + action_funcs = { + 'start': _start, + 'stop': _stop, + 'restart': _restart, + } + + def _get_action_func(self): + """ Get the function for the specified action. + + :return: The function object corresponding to the specified + action. + :raises DaemonRunnerInvalidActionError: if the action is + unknown. + + The action is specified by the `action` attribute, which is set + during `parse_args`. + + """ + try: + func = self.action_funcs[self.action] + except KeyError: + error = DaemonRunnerInvalidActionError( + "Unknown action: {action!r}".format( + action=self.action)) + raise error + return func + + def do_action(self): + """ Perform the requested action. + + :return: ``None``. + + The action is specified by the `action` attribute, which is set + during `parse_args`. + + """ + func = self._get_action_func() + func(self) + + +def emit_message(message, stream=None): + """ Emit a message to the specified stream (default `sys.stderr`). """ + if stream is None: + stream = sys.stderr + stream.write("{message}\n".format(message=message)) + stream.flush() + + +def make_pidlockfile(path, acquire_timeout): + """ Make a PIDLockFile instance with the given filesystem path. """ + if not isinstance(path, basestring): + error = ValueError("Not a filesystem path: {path!r}".format( + path=path)) + raise error + if not os.path.isabs(path): + error = ValueError("Not an absolute path: {path!r}".format( + path=path)) + raise error + lockfile = pidfile.TimeoutPIDLockFile(path, acquire_timeout) + + return lockfile + + +def is_pidfile_stale(pidfile): + """ Determine whether a PID file is stale. + + :return: ``True`` iff the PID file is stale; otherwise ``False``. + + The PID file is “stale” if its contents are valid but do not + match the PID of a currently-running process. + + """ + result = False + + pidfile_pid = pidfile.read_pid() + if pidfile_pid is not None: + try: + os.kill(pidfile_pid, signal.SIG_DFL) + except ProcessLookupError: + # The specified PID does not exist. + result = True + except OSError as exc: + if exc.errno == errno.ESRCH: + # Under Python 2, process lookup error is an OSError. + # The specified PID does not exist. + result = True + + return result + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/doc/CREDITS b/scripts/external_libs/python-daemon-2.0.5/doc/CREDITS new file mode 100644 index 00000000..feb65d5e --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/doc/CREDITS @@ -0,0 +1,53 @@ +Credits for contributors to ‘python-daemon’ +########################################### + +:Updated: 2014-12-23 + +The ‘python-daemon’ library is the work of many contributors. + + +Primary developers +================== + +The library has been maintained over the years by: + +* Ben Finney <ben+python@benfinney.id.au> +* Robert Niederreiter +* Jens Klein + + +Precursors +========== + +The library code base is inherited from prior work by: + +* Chad J. Schroeder +* Clark Evans +* Noah Spurrier +* Jürgen Hermann + + +Additional contributors +======================= + +People who have also contributed substantial improvements: + + + +.. + This is free software: you may copy, modify, and/or distribute this work + under the terms of the Apache License version 2.0 as published by the + Apache Software Foundation. + No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +.. + Local variables: + coding: utf-8 + mode: text + mode: rst + time-stamp-format: "%:y-%02m-%02d" + time-stamp-start: "^:Updated:[ ]+" + time-stamp-end: "$" + time-stamp-line-limit: 20 + End: + vim: fileencoding=utf-8 filetype=rst : diff --git a/scripts/external_libs/python-daemon-2.0.5/doc/FAQ b/scripts/external_libs/python-daemon-2.0.5/doc/FAQ new file mode 100644 index 00000000..1fcc4658 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/doc/FAQ @@ -0,0 +1,156 @@ +‘python-daemon’ Frequently Asked Questions +########################################## + +:Author: Ben Finney <ben+python@benfinney.id.au> +:Updated: 2015-01-10 + +.. contents:: +.. + 1 General + 1.1 What is the purpose of the ‘python-daemon’ library? + 1.2 How can I run a service communicating with a separate daemon process? + 2 Security + 2.1 Why is the umask set to 0 by default? + 3 File descriptors + 3.1 Why does the output stop after opening the daemon context? + 3.2 How can I preserve a ‘logging’ handler's file descriptor? + +General +======= + +What is the purpose of the ‘python-daemon’ library? +--------------------------------------------------- + +The ‘python-daemon’ library has a deliberately narrow focus: that of +being a reference implementation for `PEP 3143`_, “Standard daemon +process library”. + +.. _`PEP 3143`: http://www.python.org/dev/peps/pep-3143 + +How can I run a service communicating with a separate daemon process? +--------------------------------------------------------------------- + +As specified in `PEP 3143`_, the ‘python-daemon’ library is +specifically focussed on the goal of having the *current running +program* become a well-behaved Unix daemon process. This leaves open +the question of how this program is started, or about multiple +programs interacting. As detailed in PEP 3143: + + A daemon is not a service + + There is a related concept in many systems, called a “service”. A + service differs from the model in this PEP, in that rather than + having the *current* program continue to run as a daemon process, + a service starts an *additional* process to run in the background, + and the current process communicates with that additional process + via some defined channels. + + The Unix-style daemon model in this PEP can be used, among other + things, to implement the background-process part of a service; but + this PEP does not address the other aspects of setting up and + managing a service. + +A possible starting point for such a “service” model of execution is +in a `message from 2009-01-30`_ to the ``python-ideas`` forum. + +.. _`message from 2009-01-30`: http://mail.python.org/pipermail/python-ideas/2009-January/002606.html + + +Security +======== + +Why is the umask set to 0 by default? +------------------------------------- + +A daemon should not rely on the parent process's umask value, which is +beyond its control and may prevent creating a file with the required +access mode. So when the daemon context opens, the umask is set to an +explicit known value. + +If the conventional value of 0 is too open, consider setting a value +such as 0o022, 0o027, 0o077, or another specific value. Otherwise, +ensure the daemon creates every file with an explicit access mode for +the purpose. + + +File descriptors +================ + +Why does the output stop after opening the daemon context? +---------------------------------------------------------- + +The specified behaviour in `PEP 3143`_ includes the requirement to +detach the process from the controlling terminal (to allow the process +to continue to run as a daemon), and to close all file descriptors not +known to be safe once detached (to ensure any files that continue to +be used are under the control of the daemon process). + +If you want the process to generate output via the system streams +‘sys.stdout’ and ‘sys.stderr’, set the ‘DaemonContext’'s ‘stdout’ +and/or ‘stderr’ options to a file-like object (e.g. the ‘stream’ +attribute of a ‘logging.Handler’ instance). If these objects have file +descriptors, they will be preserved when the daemon context opens. + +How can I preserve a ‘logging’ handler's file descriptor? +--------------------------------------------------------- + +The ‘DaemonContext.open’ method conforms to `PEP 3143`_ by closing all +open file descriptors, but excluding those files specified in the +‘files_preserve’ option. This option is a list of files or file +descriptors. + +The Python standard library ‘logging’ module provides log handlers +that write to streams, including to files via the ‘StreamHandler’ +class and its sub-classes. The documentation (both the online `logging +module documentation`_ and the docstrings for the code) makes no +mention of a way to get at the stream associated with a handler +object. + +However, looking at the source code for ‘StreamHandler’, in Python 2.5 +as ``/usr/lib/python2.5/logging/__init__.py``, shows a ‘stream’ +attribute that is bound to the stream object. The attribute is not +marked private (i.e. it is not named with a leading underscore), so we +can presume it is part of the public API. + +That attribute can then be used to specify that a logging handler's +file descriptor should, when the ‘DaemonContext’ opens, be excluded +from closure:: + + import logging + import daemon + + # any subclass of StreamHandler should provide the ‘stream’ attribute. + lh = logging.handlers.TimedRotatingFileHandler( + "/var/log/foo.log", + # … + ) + + # … do some logging and other activity … + + daemon_context = daemon.DaemonContext() + daemon_context.files_preserve = [lh.stream] + + daemon_context.open() + + # … continue as a daemon process … + +.. _`logging module documentation`: http://docs.python.org/library/logging + + +.. + This is free software: you may copy, modify, and/or distribute this work + under the terms of the Apache License version 2.0 as published by the + Apache Software Foundation. + No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +.. + Local variables: + coding: utf-8 + mode: text + mode: rst + time-stamp-format: "%:y-%02m-%02d" + time-stamp-start: "^:Updated:[ ]+" + time-stamp-end: "$" + time-stamp-line-limit: 20 + End: + vim: fileencoding=utf-8 filetype=rst : diff --git a/scripts/external_libs/python-daemon-2.0.5/doc/TODO b/scripts/external_libs/python-daemon-2.0.5/doc/TODO new file mode 100644 index 00000000..81b41481 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/doc/TODO @@ -0,0 +1,95 @@ +TODO for ‘python-daemon’ library +################################ + +:Updated: 2015-01-10 + +======= +PENDING +======= + +Tests +===== + +Libraries +========= + +* Evaluate switching to ‘flufl.lock’ library for PID lockfile behaviour + <http://pypi.python.org/pypi/flufl.lock>_. + +Features +======== + +Important +--------- + +Wishlist +-------- + +* Allow specification of a syslog service name to log as (default: + output to stdout and stderr, not syslog). + +Documentation +============= + +Standard library inclusion +========================== + + +==== +DONE +==== + +* Convert to Python 2 and Python 3 compatible code base. + +* Work correctly with current ‘lockfile’ library (0.10 or later). + +* Write full unit tests for every new or changed behaviour at time of + commit. + +* Detect whether started by another process that handles + daemonisation, such as ‘inetd’, and behave appropriately. + +* Detach to new process and session group. + +* Allow specification of working directory (default: '/'). + +* Allow specification of umask (default: 0o000). + +* Drop ‘suid’ and ‘sgid’ privileges if set. + +* Close all open file handles. + +* Re-open stdin, stdout, stderr to user-specified files. + +* Default re-open stdin, stdout, stderr to ‘/dev/null’. + +* Allow specification of a non-root user and group to drop to, if + started as ‘root’ (default: no change of user or group). + +* Implement context manager protocol for daemon context. + +* Allow specification of PID file with its own context manager + (default: no PID file). + +* Full docstrings for functions, classes, and modules. + +* PEP 3143 for adding this library to the Python standard library. + + +.. + This is free software: you may copy, modify, and/or distribute this work + under the terms of the Apache License version 2.0 as published by the + Apache Software Foundation. + No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +.. + Local variables: + coding: utf-8 + mode: text + mode: rst + time-stamp-format: "%:y-%02m-%02d" + time-stamp-start: "^:Updated:[ ]+" + time-stamp-end: "$" + time-stamp-line-limit: 20 + End: + vim: fileencoding=utf-8 filetype=rst : diff --git a/scripts/external_libs/python-daemon-2.0.5/doc/hacking.txt b/scripts/external_libs/python-daemon-2.0.5/doc/hacking.txt new file mode 100644 index 00000000..9484dbd0 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/doc/hacking.txt @@ -0,0 +1,180 @@ +Developer's guide +################# + +:Author: Ben Finney <ben+python@benfinney.id.au> +:Updated: 2014-11-28 + + +Project layout +============== + +:: + + ./ Top level of source tree + doc/ Project documentation + bin/ Executable programs + daemon/ Main ‘daemon’ library + test/ Unit tests + + +Code style +========== + +Python +------ + +All Python code should conform to the guidelines in PEP8_. In +particular: + +* Indent each level using 4 spaces (``U+0020 SPACE``), and no TABs + (``U+0008 CHARACTER TABULATION``). + +* Name modules in lower case, ``multiplewordslikethis``. + +* Name classes in title case, ``MultipleWordsLikeThis``. + +* Name functions, instances and other variables in lower case, + ``multiple_words_like_this``. + +* Every module, class, and function has a Python doc string explaining + its purpose and API. + + *Exception*: Functions whose purpose and API are mandated by Python + itself (dunder-named methods) do not need a doc string. + +* Doc strings are written as triple-quoted strings. + + * The text of the doc string is marked up with reStructuredText. + + * The first line is a one-line synopsis of the object. This summary + line appears on the same line as the opening triple-quote, + separated by a single space. + + * Further lines, if needed, are separated from the first by one + blank line. + + * The synopsis is separated by one space from the opening + triple-quote; this causes it to appear four columns past the + beginning of the line. All subsequent lines are indented at least + four columns also. + + * The synopsis is followed by a reStructuredText field list. The + field names are: “param foo” for each parameter (where “foo” is + the parameter name), and “return” for the return value. The field + values describe the purpose of each. + + * The closing triple-quote appears on a separate line. + + Example:: + + def frobnicate(spam, algorithm="dv"): + """ Perform frobnication on ``spam``. + + :param spam: A travortionate (as a sequence of strings). + :param algorithm: The name of the algorithm to use for + frobnicating the travortionate. + :return: The frobnicated travortionate, if it is + non-empty; otherwise None. + + The frobnication is done by the Dietzel-Venkman algorithm, + and optimises for the case where ``spam`` is freebled and + agglutinative. + + """ + spagnify(spam) + # … + +* All ``import`` statements appear at the top of the module. + +* Each ``import`` statement imports a single module, or multiple names + from a single module. + + Example:: + + import sys + import os + from spam import foo, bar, baz + +.. _PEP8: http://www.python.org/dev/peps/pep-0008/ + +Additional style guidelines: + +* All text files (including program code) are encoded in UTF-8. + +* A page break (``U+000C FORM FEED``) whitespace character is used + within a module to break up semantically separate areas of the + module. + +* Editor hints for Emacs and Vim appear in a comment block at the + file's end:: + + + # Local variables: + # coding: utf-8 + # mode: python + # End: + # vim: fileencoding=utf-8 filetype=python : + + +Unit tests +========== + +All code should aim for 100% coverage by unit tests. New code, or +changes to existing code, will only be considered for inclusion in the +development tree when accompanied by corresponding additions or +changes to the unit tests. + +Test-driven development +----------------------- + +Where possible, practice test-driven development to implement program +code. + +* During a development session, maintain a separate window or terminal + with the unit test suite for the project running continuously, or + automatically every few seconds. + +* Any time a test is failing, the only valid change is to make all + tests pass. + +* Develop new interface features (changes to the program unit's + behaviour) only when all current tests pass. + +* Refactor as needed, but only when all tests pass. + + * Refactoring is any change to the code which does not alter its + interface or expected behaviour, such as performance + optimisations, readability improvements, modularisation + improvements etc. + +* Develop new or changed program behaviour by: + + * *First* write a single, specific test case for that new behaviour, + then watch the test fail in the absence of the desired behaviour. + + * Implement the minimum necessary change to satisfy the failing + test. Continue until all tests pass again, then stop making + functional changes. + + * Once all tests (including the new test) pass, consider refactoring + the code and the tests immediately, then ensure all the tests pass + again after any changes. + + * Iterate for each incremental change in interface or behaviour. + +Test-driven development is not absolutely necessary, but is the +simplest, most direct way to generate the kind of program changes +accompanied by unit tests that are necessary for inclusion in the +project. + + +.. + Local variables: + coding: utf-8 + mode: rst + time-stamp-format: "%:y-%02m-%02d" + time-stamp-start: "^:Updated:[ ]+" + time-stamp-end: "$" + time-stamp-line-limit: 20 + End: + vim: fileencoding=utf-8 filetype=rst : diff --git a/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/PKG-INFO b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/PKG-INFO new file mode 100644 index 00000000..fd81f509 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/PKG-INFO @@ -0,0 +1,38 @@ +Metadata-Version: 1.1 +Name: python-daemon +Version: 2.0.5 +Summary: Library to implement a well-behaved Unix daemon process. +Home-page: https://alioth.debian.org/projects/python-daemon/ +Author: Ben Finney +Author-email: ben+python@benfinney.id.au +License: Apache-2 +Description: This library implements the well-behaved daemon specification of + :pep:`3143`, “Standard daemon process library”. + + A well-behaved Unix daemon process is tricky to get right, but the + required steps are much the same for every daemon program. A + `DaemonContext` instance holds the behaviour and configured + process environment for the program; use the instance as a context + manager to enter a daemon state. + + Simple example of usage:: + + import daemon + + from spam import do_main_program + + with daemon.DaemonContext(): + do_main_program() + + Customisation of the steps to become a daemon is available by + setting options on the `DaemonContext` instance; see the + documentation for that class for each option. +Keywords: daemon,fork,unix +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: POSIX +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Intended Audience :: Developers +Classifier: Topic :: Software Development :: Libraries :: Python Modules diff --git a/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/SOURCES.txt b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/SOURCES.txt new file mode 100644 index 00000000..6e176719 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/SOURCES.txt @@ -0,0 +1,30 @@ +ChangeLog +LICENSE.ASF-2 +LICENSE.GPL-3 +MANIFEST.in +setup.cfg +setup.py +test_version.py +version.py +daemon/__init__.py +daemon/_metadata.py +daemon/daemon.py +daemon/pidfile.py +daemon/runner.py +doc/CREDITS +doc/FAQ +doc/TODO +doc/hacking.txt +python_daemon.egg-info/PKG-INFO +python_daemon.egg-info/SOURCES.txt +python_daemon.egg-info/dependency_links.txt +python_daemon.egg-info/not-zip-safe +python_daemon.egg-info/requires.txt +python_daemon.egg-info/top_level.txt +python_daemon.egg-info/version_info.json +test/__init__.py +test/scaffold.py +test/test_daemon.py +test/test_metadata.py +test/test_pidfile.py +test/test_runner.py
\ No newline at end of file diff --git a/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/dependency_links.txt b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/not-zip-safe b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/not-zip-safe new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/requires.txt b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/requires.txt new file mode 100644 index 00000000..d1496b02 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/requires.txt @@ -0,0 +1,3 @@ +setuptools +docutils +lockfile >=0.10 diff --git a/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/top_level.txt b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/top_level.txt new file mode 100644 index 00000000..28e3ee0c --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/top_level.txt @@ -0,0 +1 @@ +daemon diff --git a/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/version_info.json b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/version_info.json new file mode 100644 index 00000000..bac1b84f --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/python_daemon.egg-info/version_info.json @@ -0,0 +1,6 @@ +{ + "release_date": "2015-02-02", + "version": "2.0.5", + "maintainer": "Ben Finney <ben+python@benfinney.id.au>", + "body": "* Refine compatibility of exceptions for file operations.\n* Specify the text encoding when opening the changelog file.\n" +}
\ No newline at end of file diff --git a/scripts/external_libs/python-daemon-2.0.5/setup.cfg b/scripts/external_libs/python-daemon-2.0.5/setup.cfg new file mode 100644 index 00000000..9d3d2c02 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/setup.cfg @@ -0,0 +1,11 @@ +[aliases] +distribute = register sdist bdist_wheel upload + +[bdist_wheel] +universal = true + +[egg_info] +tag_svn_revision = 0 +tag_date = 0 +tag_build = + diff --git a/scripts/external_libs/python-daemon-2.0.5/setup.py b/scripts/external_libs/python-daemon-2.0.5/setup.py new file mode 100644 index 00000000..16a6a6a6 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/setup.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +# setup.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au> +# Copyright © 2008 Robert Niederreiter, Jens Klein +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; version 3 of that license or any later version. +# No warranty expressed or implied. See the file ‘LICENSE.GPL-3’ for details. + +""" Distribution setup for ‘python-daemon’ library. """ + +from __future__ import (absolute_import, unicode_literals) + +import sys +import os +import os.path +import pydoc +import distutils.util + +from setuptools import (setup, find_packages) + +import version + + +fromlist_expects_type = str +if sys.version_info < (3, 0): + fromlist_expects_type = bytes + + +main_module_name = 'daemon' +main_module_fromlist = list(map(fromlist_expects_type, [ + '_metadata'])) +main_module = __import__( + main_module_name, + level=0, fromlist=main_module_fromlist) +metadata = main_module._metadata + +(synopsis, long_description) = pydoc.splitdoc(pydoc.getdoc(main_module)) + +version_info = metadata.get_distribution_version_info() +version_string = version_info['version'] + +(maintainer_name, maintainer_email) = metadata.parse_person_field( + version_info['maintainer']) + + +setup( + name=metadata.distribution_name, + version=version_string, + packages=find_packages(exclude=["test"]), + cmdclass={ + "write_version_info": version.WriteVersionInfoCommand, + "egg_info": version.EggInfoCommand, + }, + + # Setuptools metadata. + maintainer=maintainer_name, + maintainer_email=maintainer_email, + zip_safe=False, + setup_requires=[ + "docutils", + ], + test_suite="unittest2.collector", + tests_require=[ + "unittest2 >=0.6", + "testtools", + "testscenarios >=0.4", + "mock >=1.0", + "docutils", + ], + install_requires=[ + "setuptools", + "docutils", + "lockfile >=0.10", + ], + + # PyPI metadata. + author=metadata.author_name, + author_email=metadata.author_email, + description=synopsis, + license=metadata.license, + keywords="daemon fork unix".split(), + url=metadata.url, + long_description=long_description, + classifiers=[ + # Reference: http://pypi.python.org/pypi?%3Aaction=list_classifiers + "Development Status :: 4 - Beta", + "License :: OSI Approved :: Apache Software License", + "Operating System :: POSIX", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + ) + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/test/__init__.py b/scripts/external_libs/python-daemon-2.0.5/test/__init__.py new file mode 100644 index 00000000..398519f1 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/test/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# +# test/__init__.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Unit test suite for ‘daemon’ package. + """ + +from __future__ import (absolute_import, unicode_literals) + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/test/scaffold.py b/scripts/external_libs/python-daemon-2.0.5/test/scaffold.py new file mode 100644 index 00000000..9a4f1150 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/test/scaffold.py @@ -0,0 +1,322 @@ +# -*- coding: utf-8 -*- + +# test/scaffold.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2007–2015 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Scaffolding for unit test modules. + """ + +from __future__ import (absolute_import, unicode_literals) + +import unittest +import doctest +import logging +import os +import sys +import operator +import textwrap +from copy import deepcopy +import functools + +try: + # Python 2 has both ‘str’ (bytes) and ‘unicode’ (text). + basestring = basestring + unicode = unicode +except NameError: + # Python 3 names the Unicode data type ‘str’. + basestring = str + unicode = str + +import testscenarios +import testtools.testcase + + +test_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(test_dir) +if not test_dir in sys.path: + sys.path.insert(1, test_dir) +if not parent_dir in sys.path: + sys.path.insert(1, parent_dir) + +# Disable all but the most critical logging messages. +logging.disable(logging.CRITICAL) + + +def get_function_signature(func): + """ Get the function signature as a mapping of attributes. + + :param func: The function object to interrogate. + :return: A mapping of the components of a function signature. + + The signature is constructed as a mapping: + + * 'name': The function's defined name. + * 'arg_count': The number of arguments expected by the function. + * 'arg_names': A sequence of the argument names, as strings. + * 'arg_defaults': A sequence of the default values for the arguments. + * 'va_args': The name bound to remaining positional arguments. + * 'va_kw_args': The name bound to remaining keyword arguments. + + """ + try: + # Python 3 function attributes. + func_code = func.__code__ + func_defaults = func.__defaults__ + except AttributeError: + # Python 2 function attributes. + func_code = func.func_code + func_defaults = func.func_defaults + + arg_count = func_code.co_argcount + arg_names = func_code.co_varnames[:arg_count] + + arg_defaults = {} + if func_defaults is not None: + arg_defaults = dict( + (name, value) + for (name, value) in + zip(arg_names[::-1], func_defaults[::-1])) + + signature = { + 'name': func.__name__, + 'arg_count': arg_count, + 'arg_names': arg_names, + 'arg_defaults': arg_defaults, + } + + non_pos_names = list(func_code.co_varnames[arg_count:]) + COLLECTS_ARBITRARY_POSITIONAL_ARGS = 0x04 + if func_code.co_flags & COLLECTS_ARBITRARY_POSITIONAL_ARGS: + signature['var_args'] = non_pos_names.pop(0) + COLLECTS_ARBITRARY_KEYWORD_ARGS = 0x08 + if func_code.co_flags & COLLECTS_ARBITRARY_KEYWORD_ARGS: + signature['var_kw_args'] = non_pos_names.pop(0) + + return signature + + +def format_function_signature(func): + """ Format the function signature as printable text. + + :param func: The function object to interrogate. + :return: A formatted text representation of the function signature. + + The signature is rendered a text; for example:: + + foo(spam, eggs, ham=True, beans=None, *args, **kwargs) + + """ + signature = get_function_signature(func) + + args_text = [] + for arg_name in signature['arg_names']: + if arg_name in signature['arg_defaults']: + arg_text = "{name}={value!r}".format( + name=arg_name, value=signature['arg_defaults'][arg_name]) + else: + arg_text = "{name}".format( + name=arg_name) + args_text.append(arg_text) + if 'var_args' in signature: + args_text.append("*{var_args}".format(signature)) + if 'var_kw_args' in signature: + args_text.append("**{var_kw_args}".format(signature)) + signature_args_text = ", ".join(args_text) + + func_name = signature['name'] + signature_text = "{name}({args})".format( + name=func_name, args=signature_args_text) + + return signature_text + + +class TestCase(testtools.testcase.TestCase): + """ Test case behaviour. """ + + def failUnlessOutputCheckerMatch(self, want, got, msg=None): + """ Fail unless the specified string matches the expected. + + :param want: The desired output pattern. + :param got: The actual text to match. + :param msg: A message to prefix on the failure message. + :return: ``None``. + :raises self.failureException: If the text does not match. + + Fail the test unless ``want`` matches ``got``, as determined by + a ``doctest.OutputChecker`` instance. This is not an equality + check, but a pattern match according to the ``OutputChecker`` + rules. + + """ + checker = doctest.OutputChecker() + want = textwrap.dedent(want) + source = "" + example = doctest.Example(source, want) + got = textwrap.dedent(got) + checker_optionflags = functools.reduce(operator.or_, [ + doctest.ELLIPSIS, + ]) + if not checker.check_output(want, got, checker_optionflags): + if msg is None: + diff = checker.output_difference( + example, got, checker_optionflags) + msg = "\n".join([ + "Output received did not match expected output", + "{diff}", + ]).format( + diff=diff) + raise self.failureException(msg) + + assertOutputCheckerMatch = failUnlessOutputCheckerMatch + + def failUnlessFunctionInTraceback(self, traceback, function, msg=None): + """ Fail if the function is not in the traceback. + + :param traceback: The traceback object to interrogate. + :param function: The function object to match. + :param msg: A message to prefix on the failure message. + :return: ``None``. + + :raises self.failureException: If the function is not in the + traceback. + + Fail the test if the function ``function`` is not at any of the + levels in the traceback object ``traceback``. + + """ + func_in_traceback = False + expected_code = function.func_code + current_traceback = traceback + while current_traceback is not None: + if expected_code is current_traceback.tb_frame.f_code: + func_in_traceback = True + break + current_traceback = current_traceback.tb_next + + if not func_in_traceback: + if msg is None: + msg = ( + "Traceback did not lead to original function" + " {function}" + ).format( + function=function) + raise self.failureException(msg) + + assertFunctionInTraceback = failUnlessFunctionInTraceback + + def failUnlessFunctionSignatureMatch(self, first, second, msg=None): + """ Fail if the function signatures do not match. + + :param first: The first function to compare. + :param second: The second function to compare. + :param msg: A message to prefix to the failure message. + :return: ``None``. + + :raises self.failureException: If the function signatures do + not match. + + Fail the test if the function signature does not match between + the ``first`` function and the ``second`` function. + + The function signature includes: + + * function name, + + * count of named parameters, + + * sequence of named parameters, + + * default values of named parameters, + + * collector for arbitrary positional arguments, + + * collector for arbitrary keyword arguments. + + """ + first_signature = get_function_signature(first) + second_signature = get_function_signature(second) + + if first_signature != second_signature: + if msg is None: + first_signature_text = format_function_signature(first) + second_signature_text = format_function_signature(second) + msg = (textwrap.dedent("""\ + Function signatures do not match: + {first!r} != {second!r} + Expected: + {first_text} + Got: + {second_text}""") + ).format( + first=first_signature, + first_text=first_signature_text, + second=second_signature, + second_text=second_signature_text, + ) + raise self.failureException(msg) + + assertFunctionSignatureMatch = failUnlessFunctionSignatureMatch + + +class TestCaseWithScenarios(testscenarios.WithScenarios, TestCase): + """ Test cases run per scenario. """ + + +class Exception_TestCase(TestCaseWithScenarios): + """ Test cases for exception classes. """ + + def test_exception_instance(self): + """ Exception instance should be created. """ + self.assertIsNot(self.instance, None) + + def test_exception_types(self): + """ Exception instance should match expected types. """ + for match_type in self.types: + self.assertIsInstance(self.instance, match_type) + + +def make_exception_scenarios(scenarios): + """ Make test scenarios for exception classes. + + :param scenarios: Sequence of scenarios. + :return: List of scenarios with additional mapping entries. + + Use this with `testscenarios` to adapt `Exception_TestCase`_ for + any exceptions that need testing. + + Each scenario is a tuple (`name`, `map`) where `map` is a mapping + of attributes to be applied to each test case. Attributes map must + contain items for: + + :key exc_type: + The exception type to be tested. + :key min_args: + The minimum argument count for the exception instance + initialiser. + :key types: + Sequence of types that should be superclasses of each + instance of the exception type. + + """ + updated_scenarios = deepcopy(scenarios) + for (name, scenario) in updated_scenarios: + args = (None,) * scenario['min_args'] + scenario['args'] = args + instance = scenario['exc_type'](*args) + scenario['instance'] = instance + + return updated_scenarios + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/test/test_daemon.py b/scripts/external_libs/python-daemon-2.0.5/test/test_daemon.py new file mode 100644 index 00000000..a911858a --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/test/test_daemon.py @@ -0,0 +1,1744 @@ +# -*- coding: utf-8 -*- +# +# test/test_daemon.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Unit test for ‘daemon’ module. + """ + +from __future__ import (absolute_import, unicode_literals) + +import os +import sys +import tempfile +import resource +import errno +import signal +import socket +from types import ModuleType +import collections +import functools +try: + # Standard library of Python 2.7 and later. + from io import StringIO +except ImportError: + # Standard library of Python 2.6 and earlier. + from StringIO import StringIO + +import mock + +from . import scaffold +from .scaffold import (basestring, unicode) +from .test_pidfile import ( + FakeFileDescriptorStringIO, + setup_pidfile_fixtures, + ) + +import daemon + + +class ModuleExceptions_TestCase(scaffold.Exception_TestCase): + """ Test cases for module exception classes. """ + + scenarios = scaffold.make_exception_scenarios([ + ('daemon.daemon.DaemonError', dict( + exc_type = daemon.daemon.DaemonError, + min_args = 1, + types = [Exception], + )), + ('daemon.daemon.DaemonOSEnvironmentError', dict( + exc_type = daemon.daemon.DaemonOSEnvironmentError, + min_args = 1, + types = [daemon.daemon.DaemonError, OSError], + )), + ('daemon.daemon.DaemonProcessDetachError', dict( + exc_type = daemon.daemon.DaemonProcessDetachError, + min_args = 1, + types = [daemon.daemon.DaemonError, OSError], + )), + ]) + + +def setup_daemon_context_fixtures(testcase): + """ Set up common test fixtures for DaemonContext test case. + + :param testcase: A ``TestCase`` instance to decorate. + :return: ``None``. + + Decorate the `testcase` with fixtures for tests involving + `DaemonContext`. + + """ + setup_streams_fixtures(testcase) + + setup_pidfile_fixtures(testcase) + + testcase.fake_pidfile_path = tempfile.mktemp() + testcase.mock_pidlockfile = mock.MagicMock() + testcase.mock_pidlockfile.path = testcase.fake_pidfile_path + + testcase.daemon_context_args = dict( + stdin=testcase.stream_files_by_name['stdin'], + stdout=testcase.stream_files_by_name['stdout'], + stderr=testcase.stream_files_by_name['stderr'], + ) + testcase.test_instance = daemon.DaemonContext( + **testcase.daemon_context_args) + +fake_default_signal_map = object() + +@mock.patch.object( + daemon.daemon, "is_detach_process_context_required", + new=(lambda: True)) +@mock.patch.object( + daemon.daemon, "make_default_signal_map", + new=(lambda: fake_default_signal_map)) +@mock.patch.object(os, "setgid", new=(lambda x: object())) +@mock.patch.object(os, "setuid", new=(lambda x: object())) +class DaemonContext_BaseTestCase(scaffold.TestCase): + """ Base class for DaemonContext test case classes. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonContext_BaseTestCase, self).setUp() + + setup_daemon_context_fixtures(self) + + +class DaemonContext_TestCase(DaemonContext_BaseTestCase): + """ Test cases for DaemonContext class. """ + + def test_instantiate(self): + """ New instance of DaemonContext should be created. """ + self.assertIsInstance( + self.test_instance, daemon.daemon.DaemonContext) + + def test_minimum_zero_arguments(self): + """ Initialiser should not require any arguments. """ + instance = daemon.daemon.DaemonContext() + self.assertIsNot(instance, None) + + def test_has_specified_chroot_directory(self): + """ Should have specified chroot_directory option. """ + args = dict( + chroot_directory=object(), + ) + expected_directory = args['chroot_directory'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_directory, instance.chroot_directory) + + def test_has_specified_working_directory(self): + """ Should have specified working_directory option. """ + args = dict( + working_directory=object(), + ) + expected_directory = args['working_directory'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_directory, instance.working_directory) + + def test_has_default_working_directory(self): + """ Should have default working_directory option. """ + args = dict() + expected_directory = "/" + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_directory, instance.working_directory) + + def test_has_specified_creation_mask(self): + """ Should have specified umask option. """ + args = dict( + umask=object(), + ) + expected_mask = args['umask'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_mask, instance.umask) + + def test_has_default_creation_mask(self): + """ Should have default umask option. """ + args = dict() + expected_mask = 0 + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_mask, instance.umask) + + def test_has_specified_uid(self): + """ Should have specified uid option. """ + args = dict( + uid=object(), + ) + expected_id = args['uid'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_id, instance.uid) + + def test_has_derived_uid(self): + """ Should have uid option derived from process. """ + args = dict() + expected_id = os.getuid() + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_id, instance.uid) + + def test_has_specified_gid(self): + """ Should have specified gid option. """ + args = dict( + gid=object(), + ) + expected_id = args['gid'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_id, instance.gid) + + def test_has_derived_gid(self): + """ Should have gid option derived from process. """ + args = dict() + expected_id = os.getgid() + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_id, instance.gid) + + def test_has_specified_detach_process(self): + """ Should have specified detach_process option. """ + args = dict( + detach_process=object(), + ) + expected_value = args['detach_process'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_value, instance.detach_process) + + def test_has_derived_detach_process(self): + """ Should have detach_process option derived from environment. """ + args = dict() + func = daemon.daemon.is_detach_process_context_required + expected_value = func() + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_value, instance.detach_process) + + def test_has_specified_files_preserve(self): + """ Should have specified files_preserve option. """ + args = dict( + files_preserve=object(), + ) + expected_files_preserve = args['files_preserve'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_files_preserve, instance.files_preserve) + + def test_has_specified_pidfile(self): + """ Should have the specified pidfile. """ + args = dict( + pidfile=object(), + ) + expected_pidfile = args['pidfile'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_pidfile, instance.pidfile) + + def test_has_specified_stdin(self): + """ Should have specified stdin option. """ + args = dict( + stdin=object(), + ) + expected_file = args['stdin'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_file, instance.stdin) + + def test_has_specified_stdout(self): + """ Should have specified stdout option. """ + args = dict( + stdout=object(), + ) + expected_file = args['stdout'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_file, instance.stdout) + + def test_has_specified_stderr(self): + """ Should have specified stderr option. """ + args = dict( + stderr=object(), + ) + expected_file = args['stderr'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_file, instance.stderr) + + def test_has_specified_signal_map(self): + """ Should have specified signal_map option. """ + args = dict( + signal_map=object(), + ) + expected_signal_map = args['signal_map'] + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_signal_map, instance.signal_map) + + def test_has_derived_signal_map(self): + """ Should have signal_map option derived from system. """ + args = dict() + expected_signal_map = daemon.daemon.make_default_signal_map() + instance = daemon.daemon.DaemonContext(**args) + self.assertEqual(expected_signal_map, instance.signal_map) + + +class DaemonContext_is_open_TestCase(DaemonContext_BaseTestCase): + """ Test cases for DaemonContext.is_open property. """ + + def test_begin_false(self): + """ Initial value of is_open should be False. """ + instance = self.test_instance + self.assertEqual(False, instance.is_open) + + def test_write_fails(self): + """ Writing to is_open should fail. """ + instance = self.test_instance + self.assertRaises( + AttributeError, + setattr, instance, 'is_open', object()) + + +class DaemonContext_open_TestCase(DaemonContext_BaseTestCase): + """ Test cases for DaemonContext.open method. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonContext_open_TestCase, self).setUp() + + self.test_instance._is_open = False + + self.mock_module_daemon = mock.MagicMock() + daemon_func_patchers = dict( + (func_name, mock.patch.object( + daemon.daemon, func_name)) + for func_name in [ + "detach_process_context", + "change_working_directory", + "change_root_directory", + "change_file_creation_mask", + "change_process_owner", + "prevent_core_dump", + "close_all_open_files", + "redirect_stream", + "set_signal_handlers", + "register_atexit_function", + ]) + for (func_name, patcher) in daemon_func_patchers.items(): + mock_func = patcher.start() + self.addCleanup(patcher.stop) + self.mock_module_daemon.attach_mock(mock_func, func_name) + + self.mock_module_daemon.attach_mock(mock.Mock(), 'DaemonContext') + + self.test_files_preserve_fds = object() + self.test_signal_handler_map = object() + daemoncontext_method_return_values = { + '_get_exclude_file_descriptors': + self.test_files_preserve_fds, + '_make_signal_handler_map': + self.test_signal_handler_map, + } + daemoncontext_func_patchers = dict( + (func_name, mock.patch.object( + daemon.daemon.DaemonContext, + func_name, + return_value=return_value)) + for (func_name, return_value) in + daemoncontext_method_return_values.items()) + for (func_name, patcher) in daemoncontext_func_patchers.items(): + mock_func = patcher.start() + self.addCleanup(patcher.stop) + self.mock_module_daemon.DaemonContext.attach_mock( + mock_func, func_name) + + def test_performs_steps_in_expected_sequence(self): + """ Should perform daemonisation steps in expected sequence. """ + instance = self.test_instance + instance.chroot_directory = object() + instance.detach_process = True + instance.pidfile = self.mock_pidlockfile + self.mock_module_daemon.attach_mock( + self.mock_pidlockfile, 'pidlockfile') + expected_calls = [ + mock.call.change_root_directory(mock.ANY), + mock.call.prevent_core_dump(), + mock.call.change_file_creation_mask(mock.ANY), + mock.call.change_working_directory(mock.ANY), + mock.call.change_process_owner(mock.ANY, mock.ANY), + mock.call.detach_process_context(), + mock.call.DaemonContext._make_signal_handler_map(), + mock.call.set_signal_handlers(mock.ANY), + mock.call.DaemonContext._get_exclude_file_descriptors(), + mock.call.close_all_open_files(exclude=mock.ANY), + mock.call.redirect_stream(mock.ANY, mock.ANY), + mock.call.redirect_stream(mock.ANY, mock.ANY), + mock.call.redirect_stream(mock.ANY, mock.ANY), + mock.call.pidlockfile.__enter__(), + mock.call.register_atexit_function(mock.ANY), + ] + instance.open() + self.mock_module_daemon.assert_has_calls(expected_calls) + + def test_returns_immediately_if_is_open(self): + """ Should return immediately if is_open property is true. """ + instance = self.test_instance + instance._is_open = True + instance.open() + self.assertEqual(0, len(self.mock_module_daemon.mock_calls)) + + def test_changes_root_directory_to_chroot_directory(self): + """ Should change root directory to `chroot_directory` option. """ + instance = self.test_instance + chroot_directory = object() + instance.chroot_directory = chroot_directory + instance.open() + self.mock_module_daemon.change_root_directory.assert_called_with( + chroot_directory) + + def test_omits_chroot_if_no_chroot_directory(self): + """ Should omit changing root directory if no `chroot_directory`. """ + instance = self.test_instance + instance.chroot_directory = None + instance.open() + self.assertFalse(self.mock_module_daemon.change_root_directory.called) + + def test_prevents_core_dump(self): + """ Should request prevention of core dumps. """ + instance = self.test_instance + instance.open() + self.mock_module_daemon.prevent_core_dump.assert_called_with() + + def test_omits_prevent_core_dump_if_prevent_core_false(self): + """ Should omit preventing core dumps if `prevent_core` is false. """ + instance = self.test_instance + instance.prevent_core = False + instance.open() + self.assertFalse(self.mock_module_daemon.prevent_core_dump.called) + + def test_closes_open_files(self): + """ Should close all open files, excluding `files_preserve`. """ + instance = self.test_instance + expected_exclude = self.test_files_preserve_fds + instance.open() + self.mock_module_daemon.close_all_open_files.assert_called_with( + exclude=expected_exclude) + + def test_changes_directory_to_working_directory(self): + """ Should change current directory to `working_directory` option. """ + instance = self.test_instance + working_directory = object() + instance.working_directory = working_directory + instance.open() + self.mock_module_daemon.change_working_directory.assert_called_with( + working_directory) + + def test_changes_creation_mask_to_umask(self): + """ Should change file creation mask to `umask` option. """ + instance = self.test_instance + umask = object() + instance.umask = umask + instance.open() + self.mock_module_daemon.change_file_creation_mask.assert_called_with( + umask) + + def test_changes_owner_to_specified_uid_and_gid(self): + """ Should change process UID and GID to `uid` and `gid` options. """ + instance = self.test_instance + uid = object() + gid = object() + instance.uid = uid + instance.gid = gid + instance.open() + self.mock_module_daemon.change_process_owner.assert_called_with( + uid, gid) + + def test_detaches_process_context(self): + """ Should request detach of process context. """ + instance = self.test_instance + instance.open() + self.mock_module_daemon.detach_process_context.assert_called_with() + + def test_omits_process_detach_if_not_required(self): + """ Should omit detach of process context if not required. """ + instance = self.test_instance + instance.detach_process = False + instance.open() + self.assertFalse(self.mock_module_daemon.detach_process_context.called) + + def test_sets_signal_handlers_from_signal_map(self): + """ Should set signal handlers according to `signal_map`. """ + instance = self.test_instance + instance.signal_map = object() + expected_signal_handler_map = self.test_signal_handler_map + instance.open() + self.mock_module_daemon.set_signal_handlers.assert_called_with( + expected_signal_handler_map) + + def test_redirects_standard_streams(self): + """ Should request redirection of standard stream files. """ + instance = self.test_instance + (system_stdin, system_stdout, system_stderr) = ( + sys.stdin, sys.stdout, sys.stderr) + (target_stdin, target_stdout, target_stderr) = ( + self.stream_files_by_name[name] + for name in ['stdin', 'stdout', 'stderr']) + expected_calls = [ + mock.call(system_stdin, target_stdin), + mock.call(system_stdout, target_stdout), + mock.call(system_stderr, target_stderr), + ] + instance.open() + self.mock_module_daemon.redirect_stream.assert_has_calls( + expected_calls, any_order=True) + + def test_enters_pidfile_context(self): + """ Should enter the PID file context manager. """ + instance = self.test_instance + instance.pidfile = self.mock_pidlockfile + instance.open() + self.mock_pidlockfile.__enter__.assert_called_with() + + def test_sets_is_open_true(self): + """ Should set the `is_open` property to True. """ + instance = self.test_instance + instance.open() + self.assertEqual(True, instance.is_open) + + def test_registers_close_method_for_atexit(self): + """ Should register the `close` method for atexit processing. """ + instance = self.test_instance + close_method = instance.close + instance.open() + self.mock_module_daemon.register_atexit_function.assert_called_with( + close_method) + + +class DaemonContext_close_TestCase(DaemonContext_BaseTestCase): + """ Test cases for DaemonContext.close method. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonContext_close_TestCase, self).setUp() + + self.test_instance._is_open = True + + def test_returns_immediately_if_not_is_open(self): + """ Should return immediately if is_open property is false. """ + instance = self.test_instance + instance._is_open = False + instance.pidfile = object() + instance.close() + self.assertFalse(self.mock_pidlockfile.__exit__.called) + + def test_exits_pidfile_context(self): + """ Should exit the PID file context manager. """ + instance = self.test_instance + instance.pidfile = self.mock_pidlockfile + instance.close() + self.mock_pidlockfile.__exit__.assert_called_with(None, None, None) + + def test_returns_none(self): + """ Should return None. """ + instance = self.test_instance + expected_result = None + result = instance.close() + self.assertIs(result, expected_result) + + def test_sets_is_open_false(self): + """ Should set the `is_open` property to False. """ + instance = self.test_instance + instance.close() + self.assertEqual(False, instance.is_open) + + +@mock.patch.object(daemon.daemon.DaemonContext, "open") +class DaemonContext_context_manager_enter_TestCase(DaemonContext_BaseTestCase): + """ Test cases for DaemonContext.__enter__ method. """ + + def test_opens_daemon_context(self, mock_func_daemoncontext_open): + """ Should open the DaemonContext. """ + instance = self.test_instance + instance.__enter__() + mock_func_daemoncontext_open.assert_called_with() + + def test_returns_self_instance(self, mock_func_daemoncontext_open): + """ Should return DaemonContext instance. """ + instance = self.test_instance + expected_result = instance + result = instance.__enter__() + self.assertIs(result, expected_result) + + +@mock.patch.object(daemon.daemon.DaemonContext, "close") +class DaemonContext_context_manager_exit_TestCase(DaemonContext_BaseTestCase): + """ Test cases for DaemonContext.__exit__ method. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonContext_context_manager_exit_TestCase, self).setUp() + + self.test_args = dict( + exc_type=object(), + exc_value=object(), + traceback=object(), + ) + + def test_closes_daemon_context(self, mock_func_daemoncontext_close): + """ Should close the DaemonContext. """ + instance = self.test_instance + args = self.test_args + instance.__exit__(**args) + mock_func_daemoncontext_close.assert_called_with() + + def test_returns_none(self, mock_func_daemoncontext_close): + """ Should return None, indicating exception was not handled. """ + instance = self.test_instance + args = self.test_args + expected_result = None + result = instance.__exit__(**args) + self.assertIs(result, expected_result) + + +class DaemonContext_terminate_TestCase(DaemonContext_BaseTestCase): + """ Test cases for DaemonContext.terminate method. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonContext_terminate_TestCase, self).setUp() + + self.test_signal = signal.SIGTERM + self.test_frame = None + self.test_args = (self.test_signal, self.test_frame) + + def test_raises_system_exit(self): + """ Should raise SystemExit. """ + instance = self.test_instance + args = self.test_args + expected_exception = SystemExit + self.assertRaises( + expected_exception, + instance.terminate, *args) + + def test_exception_message_contains_signal_number(self): + """ Should raise exception with a message containing signal number. """ + instance = self.test_instance + args = self.test_args + signal_number = self.test_signal + expected_exception = SystemExit + exc = self.assertRaises( + expected_exception, + instance.terminate, *args) + self.assertIn(unicode(signal_number), unicode(exc)) + + +class DaemonContext_get_exclude_file_descriptors_TestCase( + DaemonContext_BaseTestCase): + """ Test cases for DaemonContext._get_exclude_file_descriptors function. """ + + def setUp(self): + """ Set up test fixtures. """ + super( + DaemonContext_get_exclude_file_descriptors_TestCase, + self).setUp() + + self.test_files = { + 2: FakeFileDescriptorStringIO(), + 5: 5, + 11: FakeFileDescriptorStringIO(), + 17: None, + 23: FakeFileDescriptorStringIO(), + 37: 37, + 42: FakeFileDescriptorStringIO(), + } + for (fileno, item) in self.test_files.items(): + if hasattr(item, '_fileno'): + item._fileno = fileno + self.test_file_descriptors = set( + fd for (fd, item) in self.test_files.items() + if item is not None) + self.test_file_descriptors.update( + self.stream_files_by_name[name].fileno() + for name in ['stdin', 'stdout', 'stderr'] + ) + + def test_returns_expected_file_descriptors(self): + """ Should return expected set of file descriptors. """ + instance = self.test_instance + instance.files_preserve = list(self.test_files.values()) + expected_result = self.test_file_descriptors + result = instance._get_exclude_file_descriptors() + self.assertEqual(expected_result, result) + + def test_returns_stream_redirects_if_no_files_preserve(self): + """ Should return only stream redirects if no files_preserve. """ + instance = self.test_instance + instance.files_preserve = None + expected_result = set( + stream.fileno() + for stream in self.stream_files_by_name.values()) + result = instance._get_exclude_file_descriptors() + self.assertEqual(expected_result, result) + + def test_returns_empty_set_if_no_files(self): + """ Should return empty set if no file options. """ + instance = self.test_instance + for name in ['files_preserve', 'stdin', 'stdout', 'stderr']: + setattr(instance, name, None) + expected_result = set() + result = instance._get_exclude_file_descriptors() + self.assertEqual(expected_result, result) + + def test_omits_non_file_streams(self): + """ Should omit non-file stream attributes. """ + instance = self.test_instance + instance.files_preserve = list(self.test_files.values()) + stream_files = self.stream_files_by_name + expected_result = self.test_file_descriptors.copy() + for (pseudo_stream_name, pseudo_stream) in stream_files.items(): + test_non_file_object = object() + setattr(instance, pseudo_stream_name, test_non_file_object) + stream_fd = pseudo_stream.fileno() + expected_result.discard(stream_fd) + result = instance._get_exclude_file_descriptors() + self.assertEqual(expected_result, result) + + def test_includes_verbatim_streams_without_file_descriptor(self): + """ Should include verbatim any stream without a file descriptor. """ + instance = self.test_instance + instance.files_preserve = list(self.test_files.values()) + stream_files = self.stream_files_by_name + mock_fileno_method = mock.MagicMock( + spec=sys.__stdin__.fileno, + side_effect=ValueError) + expected_result = self.test_file_descriptors.copy() + for (pseudo_stream_name, pseudo_stream) in stream_files.items(): + test_non_fd_stream = StringIO() + if not hasattr(test_non_fd_stream, 'fileno'): + # Python < 3 StringIO doesn't have ‘fileno’ at all. + # Add a method which raises an exception. + test_non_fd_stream.fileno = mock_fileno_method + setattr(instance, pseudo_stream_name, test_non_fd_stream) + stream_fd = pseudo_stream.fileno() + expected_result.discard(stream_fd) + expected_result.add(test_non_fd_stream) + result = instance._get_exclude_file_descriptors() + self.assertEqual(expected_result, result) + + def test_omits_none_streams(self): + """ Should omit any stream attribute which is None. """ + instance = self.test_instance + instance.files_preserve = list(self.test_files.values()) + stream_files = self.stream_files_by_name + expected_result = self.test_file_descriptors.copy() + for (pseudo_stream_name, pseudo_stream) in stream_files.items(): + setattr(instance, pseudo_stream_name, None) + stream_fd = pseudo_stream.fileno() + expected_result.discard(stream_fd) + result = instance._get_exclude_file_descriptors() + self.assertEqual(expected_result, result) + + +class DaemonContext_make_signal_handler_TestCase(DaemonContext_BaseTestCase): + """ Test cases for DaemonContext._make_signal_handler function. """ + + def test_returns_ignore_for_none(self): + """ Should return SIG_IGN when None handler specified. """ + instance = self.test_instance + target = None + expected_result = signal.SIG_IGN + result = instance._make_signal_handler(target) + self.assertEqual(expected_result, result) + + def test_returns_method_for_name(self): + """ Should return method of DaemonContext when name specified. """ + instance = self.test_instance + target = 'terminate' + expected_result = instance.terminate + result = instance._make_signal_handler(target) + self.assertEqual(expected_result, result) + + def test_raises_error_for_unknown_name(self): + """ Should raise AttributeError for unknown method name. """ + instance = self.test_instance + target = 'b0gUs' + expected_error = AttributeError + self.assertRaises( + expected_error, + instance._make_signal_handler, target) + + def test_returns_object_for_object(self): + """ Should return same object for any other object. """ + instance = self.test_instance + target = object() + expected_result = target + result = instance._make_signal_handler(target) + self.assertEqual(expected_result, result) + + +class DaemonContext_make_signal_handler_map_TestCase( + DaemonContext_BaseTestCase): + """ Test cases for DaemonContext._make_signal_handler_map function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonContext_make_signal_handler_map_TestCase, self).setUp() + + self.test_instance.signal_map = { + object(): object(), + object(): object(), + object(): object(), + } + + self.test_signal_handlers = dict( + (key, object()) + for key in self.test_instance.signal_map.values()) + self.test_signal_handler_map = dict( + (key, self.test_signal_handlers[target]) + for (key, target) in self.test_instance.signal_map.items()) + + def fake_make_signal_handler(target): + return self.test_signal_handlers[target] + + func_patcher_make_signal_handler = mock.patch.object( + daemon.daemon.DaemonContext, "_make_signal_handler", + side_effect=fake_make_signal_handler) + self.mock_func_make_signal_handler = ( + func_patcher_make_signal_handler.start()) + self.addCleanup(func_patcher_make_signal_handler.stop) + + def test_returns_constructed_signal_handler_items(self): + """ Should return items as constructed via make_signal_handler. """ + instance = self.test_instance + expected_result = self.test_signal_handler_map + result = instance._make_signal_handler_map() + self.assertEqual(expected_result, result) + + +try: + FileNotFoundError +except NameError: + # Python 2 uses IOError. + FileNotFoundError = functools.partial(IOError, errno.ENOENT) + + +@mock.patch.object(os, "chdir") +class change_working_directory_TestCase(scaffold.TestCase): + """ Test cases for change_working_directory function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(change_working_directory_TestCase, self).setUp() + + self.test_directory = object() + self.test_args = dict( + directory=self.test_directory, + ) + + def test_changes_working_directory_to_specified_directory( + self, + mock_func_os_chdir): + """ Should change working directory to specified directory. """ + args = self.test_args + directory = self.test_directory + daemon.daemon.change_working_directory(**args) + mock_func_os_chdir.assert_called_with(directory) + + def test_raises_daemon_error_on_os_error( + self, + mock_func_os_chdir): + """ Should raise a DaemonError on receiving an IOError. """ + args = self.test_args + test_error = FileNotFoundError("No such directory") + mock_func_os_chdir.side_effect = test_error + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.change_working_directory, **args) + self.assertEqual(test_error, exc.__cause__) + + def test_error_message_contains_original_error_message( + self, + mock_func_os_chdir): + """ Should raise a DaemonError with original message. """ + args = self.test_args + test_error = FileNotFoundError("No such directory") + mock_func_os_chdir.side_effect = test_error + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.change_working_directory, **args) + self.assertIn(unicode(test_error), unicode(exc)) + + +@mock.patch.object(os, "chroot") +@mock.patch.object(os, "chdir") +class change_root_directory_TestCase(scaffold.TestCase): + """ Test cases for change_root_directory function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(change_root_directory_TestCase, self).setUp() + + self.test_directory = object() + self.test_args = dict( + directory=self.test_directory, + ) + + def test_changes_working_directory_to_specified_directory( + self, + mock_func_os_chdir, mock_func_os_chroot): + """ Should change working directory to specified directory. """ + args = self.test_args + directory = self.test_directory + daemon.daemon.change_root_directory(**args) + mock_func_os_chdir.assert_called_with(directory) + + def test_changes_root_directory_to_specified_directory( + self, + mock_func_os_chdir, mock_func_os_chroot): + """ Should change root directory to specified directory. """ + args = self.test_args + directory = self.test_directory + daemon.daemon.change_root_directory(**args) + mock_func_os_chroot.assert_called_with(directory) + + def test_raises_daemon_error_on_os_error_from_chdir( + self, + mock_func_os_chdir, mock_func_os_chroot): + """ Should raise a DaemonError on receiving an IOError from chdir. """ + args = self.test_args + test_error = FileNotFoundError("No such directory") + mock_func_os_chdir.side_effect = test_error + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.change_root_directory, **args) + self.assertEqual(test_error, exc.__cause__) + + def test_raises_daemon_error_on_os_error_from_chroot( + self, + mock_func_os_chdir, mock_func_os_chroot): + """ Should raise a DaemonError on receiving an OSError from chroot. """ + args = self.test_args + test_error = OSError(errno.EPERM, "No chroot for you!") + mock_func_os_chroot.side_effect = test_error + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.change_root_directory, **args) + self.assertEqual(test_error, exc.__cause__) + + def test_error_message_contains_original_error_message( + self, + mock_func_os_chdir, mock_func_os_chroot): + """ Should raise a DaemonError with original message. """ + args = self.test_args + test_error = FileNotFoundError("No such directory") + mock_func_os_chdir.side_effect = test_error + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.change_root_directory, **args) + self.assertIn(unicode(test_error), unicode(exc)) + + +@mock.patch.object(os, "umask") +class change_file_creation_mask_TestCase(scaffold.TestCase): + """ Test cases for change_file_creation_mask function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(change_file_creation_mask_TestCase, self).setUp() + + self.test_mask = object() + self.test_args = dict( + mask=self.test_mask, + ) + + def test_changes_umask_to_specified_mask(self, mock_func_os_umask): + """ Should change working directory to specified directory. """ + args = self.test_args + mask = self.test_mask + daemon.daemon.change_file_creation_mask(**args) + mock_func_os_umask.assert_called_with(mask) + + def test_raises_daemon_error_on_os_error_from_chdir( + self, + mock_func_os_umask): + """ Should raise a DaemonError on receiving an OSError from umask. """ + args = self.test_args + test_error = OSError(errno.EINVAL, "Whatchoo talkin' 'bout?") + mock_func_os_umask.side_effect = test_error + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.change_file_creation_mask, **args) + self.assertEqual(test_error, exc.__cause__) + + def test_error_message_contains_original_error_message( + self, + mock_func_os_umask): + """ Should raise a DaemonError with original message. """ + args = self.test_args + test_error = FileNotFoundError("No such directory") + mock_func_os_umask.side_effect = test_error + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.change_file_creation_mask, **args) + self.assertIn(unicode(test_error), unicode(exc)) + + +@mock.patch.object(os, "setgid") +@mock.patch.object(os, "setuid") +class change_process_owner_TestCase(scaffold.TestCase): + """ Test cases for change_process_owner function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(change_process_owner_TestCase, self).setUp() + + self.test_uid = object() + self.test_gid = object() + self.test_args = dict( + uid=self.test_uid, + gid=self.test_gid, + ) + + def test_changes_gid_and_uid_in_order( + self, + mock_func_os_setuid, mock_func_os_setgid): + """ Should change process GID and UID in correct order. + + Since the process requires appropriate privilege to use + either of `setuid` or `setgid`, changing the UID must be + done last. + + """ + args = self.test_args + daemon.daemon.change_process_owner(**args) + mock_func_os_setuid.assert_called() + mock_func_os_setgid.assert_called() + + def test_changes_group_id_to_gid( + self, + mock_func_os_setuid, mock_func_os_setgid): + """ Should change process GID to specified value. """ + args = self.test_args + gid = self.test_gid + daemon.daemon.change_process_owner(**args) + mock_func_os_setgid.assert_called(gid) + + def test_changes_user_id_to_uid( + self, + mock_func_os_setuid, mock_func_os_setgid): + """ Should change process UID to specified value. """ + args = self.test_args + uid = self.test_uid + daemon.daemon.change_process_owner(**args) + mock_func_os_setuid.assert_called(uid) + + def test_raises_daemon_error_on_os_error_from_setgid( + self, + mock_func_os_setuid, mock_func_os_setgid): + """ Should raise a DaemonError on receiving an OSError from setgid. """ + args = self.test_args + test_error = OSError(errno.EPERM, "No switching for you!") + mock_func_os_setgid.side_effect = test_error + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.change_process_owner, **args) + self.assertEqual(test_error, exc.__cause__) + + def test_raises_daemon_error_on_os_error_from_setuid( + self, + mock_func_os_setuid, mock_func_os_setgid): + """ Should raise a DaemonError on receiving an OSError from setuid. """ + args = self.test_args + test_error = OSError(errno.EPERM, "No switching for you!") + mock_func_os_setuid.side_effect = test_error + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.change_process_owner, **args) + self.assertEqual(test_error, exc.__cause__) + + def test_error_message_contains_original_error_message( + self, + mock_func_os_setuid, mock_func_os_setgid): + """ Should raise a DaemonError with original message. """ + args = self.test_args + test_error = OSError(errno.EINVAL, "Whatchoo talkin' 'bout?") + mock_func_os_setuid.side_effect = test_error + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.change_process_owner, **args) + self.assertIn(unicode(test_error), unicode(exc)) + + +RLimitResult = collections.namedtuple('RLimitResult', ['soft', 'hard']) + +fake_RLIMIT_CORE = object() + +@mock.patch.object(resource, "RLIMIT_CORE", new=fake_RLIMIT_CORE) +@mock.patch.object(resource, "setrlimit", side_effect=(lambda x, y: None)) +@mock.patch.object(resource, "getrlimit", side_effect=(lambda x: None)) +class prevent_core_dump_TestCase(scaffold.TestCase): + """ Test cases for prevent_core_dump function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(prevent_core_dump_TestCase, self).setUp() + + def test_sets_core_limit_to_zero( + self, + mock_func_resource_getrlimit, mock_func_resource_setrlimit): + """ Should set the RLIMIT_CORE resource to zero. """ + expected_resource = fake_RLIMIT_CORE + expected_limit = tuple(RLimitResult(soft=0, hard=0)) + daemon.daemon.prevent_core_dump() + mock_func_resource_getrlimit.assert_called_with(expected_resource) + mock_func_resource_setrlimit.assert_called_with( + expected_resource, expected_limit) + + def test_raises_error_when_no_core_resource( + self, + mock_func_resource_getrlimit, mock_func_resource_setrlimit): + """ Should raise DaemonError if no RLIMIT_CORE resource. """ + test_error = ValueError("Bogus platform doesn't have RLIMIT_CORE") + def fake_getrlimit(res): + if res == resource.RLIMIT_CORE: + raise test_error + else: + return None + mock_func_resource_getrlimit.side_effect = fake_getrlimit + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.prevent_core_dump) + self.assertEqual(test_error, exc.__cause__) + + +@mock.patch.object(os, "close") +class close_file_descriptor_if_open_TestCase(scaffold.TestCase): + """ Test cases for close_file_descriptor_if_open function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(close_file_descriptor_if_open_TestCase, self).setUp() + + self.fake_fd = 274 + + def test_requests_file_descriptor_close(self, mock_func_os_close): + """ Should request close of file descriptor. """ + fd = self.fake_fd + daemon.daemon.close_file_descriptor_if_open(fd) + mock_func_os_close.assert_called_with(fd) + + def test_ignores_badfd_error_on_close(self, mock_func_os_close): + """ Should ignore OSError EBADF when closing. """ + fd = self.fake_fd + test_error = OSError(errno.EBADF, "Bad file descriptor") + def fake_os_close(fd): + raise test_error + mock_func_os_close.side_effect = fake_os_close + daemon.daemon.close_file_descriptor_if_open(fd) + mock_func_os_close.assert_called_with(fd) + + def test_raises_error_if_oserror_on_close(self, mock_func_os_close): + """ Should raise DaemonError if an OSError occurs when closing. """ + fd = self.fake_fd + test_error = OSError(object(), "Unexpected error") + def fake_os_close(fd): + raise test_error + mock_func_os_close.side_effect = fake_os_close + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.close_file_descriptor_if_open, fd) + self.assertEqual(test_error, exc.__cause__) + + def test_raises_error_if_ioerror_on_close(self, mock_func_os_close): + """ Should raise DaemonError if an IOError occurs when closing. """ + fd = self.fake_fd + test_error = IOError(object(), "Unexpected error") + def fake_os_close(fd): + raise test_error + mock_func_os_close.side_effect = fake_os_close + expected_error = daemon.daemon.DaemonOSEnvironmentError + exc = self.assertRaises( + expected_error, + daemon.daemon.close_file_descriptor_if_open, fd) + self.assertEqual(test_error, exc.__cause__) + + +class maxfd_TestCase(scaffold.TestCase): + """ Test cases for module MAXFD constant. """ + + def test_positive(self): + """ Should be a positive number. """ + maxfd = daemon.daemon.MAXFD + self.assertTrue(maxfd > 0) + + def test_integer(self): + """ Should be an integer. """ + maxfd = daemon.daemon.MAXFD + self.assertEqual(int(maxfd), maxfd) + + def test_reasonably_high(self): + """ Should be reasonably high for default open files limit. + + If the system reports a limit of “infinity” on maximum + file descriptors, we still need a finite number in order + to close “all” of them. Ensure this is reasonably high + to catch most use cases. + + """ + expected_minimum = 2048 + maxfd = daemon.daemon.MAXFD + self.assertTrue( + expected_minimum <= maxfd, + msg=( + "MAXFD should be at least {minimum!r}" + " (got {maxfd!r})".format( + minimum=expected_minimum, maxfd=maxfd))) + + +fake_default_maxfd = 8 +fake_RLIMIT_NOFILE = object() +fake_RLIM_INFINITY = object() +fake_rlimit_nofile_large = 2468 + +def fake_getrlimit_nofile_soft_infinity(resource): + result = RLimitResult(soft=fake_RLIM_INFINITY, hard=object()) + if resource != fake_RLIMIT_NOFILE: + result = NotImplemented + return result + +def fake_getrlimit_nofile_hard_infinity(resource): + result = RLimitResult(soft=object(), hard=fake_RLIM_INFINITY) + if resource != fake_RLIMIT_NOFILE: + result = NotImplemented + return result + +def fake_getrlimit_nofile_hard_large(resource): + result = RLimitResult(soft=object(), hard=fake_rlimit_nofile_large) + if resource != fake_RLIMIT_NOFILE: + result = NotImplemented + return result + +@mock.patch.object(daemon.daemon, "MAXFD", new=fake_default_maxfd) +@mock.patch.object(resource, "RLIMIT_NOFILE", new=fake_RLIMIT_NOFILE) +@mock.patch.object(resource, "RLIM_INFINITY", new=fake_RLIM_INFINITY) +@mock.patch.object( + resource, "getrlimit", + side_effect=fake_getrlimit_nofile_hard_large) +class get_maximum_file_descriptors_TestCase(scaffold.TestCase): + """ Test cases for get_maximum_file_descriptors function. """ + + def test_returns_system_hard_limit(self, mock_func_resource_getrlimit): + """ Should return process hard limit on number of files. """ + expected_result = fake_rlimit_nofile_large + result = daemon.daemon.get_maximum_file_descriptors() + self.assertEqual(expected_result, result) + + def test_returns_module_default_if_hard_limit_infinity( + self, mock_func_resource_getrlimit): + """ Should return module MAXFD if hard limit is infinity. """ + mock_func_resource_getrlimit.side_effect = ( + fake_getrlimit_nofile_hard_infinity) + expected_result = fake_default_maxfd + result = daemon.daemon.get_maximum_file_descriptors() + self.assertEqual(expected_result, result) + + +def fake_get_maximum_file_descriptors(): + return fake_default_maxfd + +@mock.patch.object(resource, "RLIMIT_NOFILE", new=fake_RLIMIT_NOFILE) +@mock.patch.object(resource, "RLIM_INFINITY", new=fake_RLIM_INFINITY) +@mock.patch.object( + resource, "getrlimit", + new=fake_getrlimit_nofile_soft_infinity) +@mock.patch.object( + daemon.daemon, "get_maximum_file_descriptors", + new=fake_get_maximum_file_descriptors) +@mock.patch.object(daemon.daemon, "close_file_descriptor_if_open") +class close_all_open_files_TestCase(scaffold.TestCase): + """ Test cases for close_all_open_files function. """ + + def test_requests_all_open_files_to_close( + self, mock_func_close_file_descriptor_if_open): + """ Should request close of all open files. """ + expected_file_descriptors = range(fake_default_maxfd) + expected_calls = [ + mock.call(fd) for fd in expected_file_descriptors] + daemon.daemon.close_all_open_files() + mock_func_close_file_descriptor_if_open.assert_has_calls( + expected_calls, any_order=True) + + def test_requests_all_but_excluded_files_to_close( + self, mock_func_close_file_descriptor_if_open): + """ Should request close of all open files but those excluded. """ + test_exclude = set([3, 7]) + args = dict( + exclude=test_exclude, + ) + expected_file_descriptors = set( + fd for fd in range(fake_default_maxfd) + if fd not in test_exclude) + expected_calls = [ + mock.call(fd) for fd in expected_file_descriptors] + daemon.daemon.close_all_open_files(**args) + mock_func_close_file_descriptor_if_open.assert_has_calls( + expected_calls, any_order=True) + + +class detach_process_context_TestCase(scaffold.TestCase): + """ Test cases for detach_process_context function. """ + + class FakeOSExit(SystemExit): + """ Fake exception raised for os._exit(). """ + + def setUp(self): + """ Set up test fixtures. """ + super(detach_process_context_TestCase, self).setUp() + + self.mock_module_os = mock.MagicMock(wraps=os) + + fake_pids = [0, 0] + func_patcher_os_fork = mock.patch.object( + os, "fork", + side_effect=iter(fake_pids)) + self.mock_func_os_fork = func_patcher_os_fork.start() + self.addCleanup(func_patcher_os_fork.stop) + self.mock_module_os.attach_mock(self.mock_func_os_fork, "fork") + + func_patcher_os_setsid = mock.patch.object(os, "setsid") + self.mock_func_os_setsid = func_patcher_os_setsid.start() + self.addCleanup(func_patcher_os_setsid.stop) + self.mock_module_os.attach_mock(self.mock_func_os_setsid, "setsid") + + def raise_os_exit(status=None): + raise self.FakeOSExit(status) + + func_patcher_os_force_exit = mock.patch.object( + os, "_exit", + side_effect=raise_os_exit) + self.mock_func_os_force_exit = func_patcher_os_force_exit.start() + self.addCleanup(func_patcher_os_force_exit.stop) + self.mock_module_os.attach_mock(self.mock_func_os_force_exit, "_exit") + + def test_parent_exits(self): + """ Parent process should exit. """ + parent_pid = 23 + self.mock_func_os_fork.side_effect = iter([parent_pid]) + self.assertRaises( + self.FakeOSExit, + daemon.daemon.detach_process_context) + self.mock_module_os.assert_has_calls([ + mock.call.fork(), + mock.call._exit(0), + ]) + + def test_first_fork_error_raises_error(self): + """ Error on first fork should raise DaemonProcessDetachError. """ + fork_errno = 13 + fork_strerror = "Bad stuff happened" + test_error = OSError(fork_errno, fork_strerror) + test_pids_iter = iter([test_error]) + + def fake_fork(): + next_item = next(test_pids_iter) + if isinstance(next_item, Exception): + raise next_item + else: + return next_item + + self.mock_func_os_fork.side_effect = fake_fork + exc = self.assertRaises( + daemon.daemon.DaemonProcessDetachError, + daemon.daemon.detach_process_context) + self.assertEqual(test_error, exc.__cause__) + self.mock_module_os.assert_has_calls([ + mock.call.fork(), + ]) + + def test_child_starts_new_process_group(self): + """ Child should start new process group. """ + daemon.daemon.detach_process_context() + self.mock_module_os.assert_has_calls([ + mock.call.fork(), + mock.call.setsid(), + ]) + + def test_child_forks_next_parent_exits(self): + """ Child should fork, then exit if parent. """ + fake_pids = [0, 42] + self.mock_func_os_fork.side_effect = iter(fake_pids) + self.assertRaises( + self.FakeOSExit, + daemon.daemon.detach_process_context) + self.mock_module_os.assert_has_calls([ + mock.call.fork(), + mock.call.setsid(), + mock.call.fork(), + mock.call._exit(0), + ]) + + def test_second_fork_error_reports_to_stderr(self): + """ Error on second fork should cause report to stderr. """ + fork_errno = 17 + fork_strerror = "Nasty stuff happened" + test_error = OSError(fork_errno, fork_strerror) + test_pids_iter = iter([0, test_error]) + + def fake_fork(): + next_item = next(test_pids_iter) + if isinstance(next_item, Exception): + raise next_item + else: + return next_item + + self.mock_func_os_fork.side_effect = fake_fork + exc = self.assertRaises( + daemon.daemon.DaemonProcessDetachError, + daemon.daemon.detach_process_context) + self.assertEqual(test_error, exc.__cause__) + self.mock_module_os.assert_has_calls([ + mock.call.fork(), + mock.call.setsid(), + mock.call.fork(), + ]) + + def test_child_forks_next_child_continues(self): + """ Child should fork, then continue if child. """ + daemon.daemon.detach_process_context() + self.mock_module_os.assert_has_calls([ + mock.call.fork(), + mock.call.setsid(), + mock.call.fork(), + ]) + + +@mock.patch("os.getppid", return_value=765) +class is_process_started_by_init_TestCase(scaffold.TestCase): + """ Test cases for is_process_started_by_init function. """ + + def test_returns_false_by_default(self, mock_func_os_getppid): + """ Should return False under normal circumstances. """ + expected_result = False + result = daemon.daemon.is_process_started_by_init() + self.assertIs(result, expected_result) + + def test_returns_true_if_parent_process_is_init( + self, mock_func_os_getppid): + """ Should return True if parent process is `init`. """ + init_pid = 1 + mock_func_os_getppid.return_value = init_pid + expected_result = True + result = daemon.daemon.is_process_started_by_init() + self.assertIs(result, expected_result) + + +class is_socket_TestCase(scaffold.TestCase): + """ Test cases for is_socket function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(is_socket_TestCase, self).setUp() + + def fake_getsockopt(level, optname, buflen=None): + result = object() + if optname is socket.SO_TYPE: + result = socket.SOCK_RAW + return result + + self.fake_socket_getsockopt_func = fake_getsockopt + + self.fake_socket_error = socket.error( + errno.ENOTSOCK, + "Socket operation on non-socket") + + self.mock_socket = mock.MagicMock(spec=socket.socket) + self.mock_socket.getsockopt.side_effect = self.fake_socket_error + + def fake_socket_fromfd(fd, family, type, proto=None): + return self.mock_socket + + func_patcher_socket_fromfd = mock.patch.object( + socket, "fromfd", + side_effect=fake_socket_fromfd) + func_patcher_socket_fromfd.start() + self.addCleanup(func_patcher_socket_fromfd.stop) + + def test_returns_false_by_default(self): + """ Should return False under normal circumstances. """ + test_fd = 23 + expected_result = False + result = daemon.daemon.is_socket(test_fd) + self.assertIs(result, expected_result) + + def test_returns_true_if_stdin_is_socket(self): + """ Should return True if `stdin` is a socket. """ + test_fd = 23 + getsockopt = self.mock_socket.getsockopt + getsockopt.side_effect = self.fake_socket_getsockopt_func + expected_result = True + result = daemon.daemon.is_socket(test_fd) + self.assertIs(result, expected_result) + + def test_returns_false_if_stdin_socket_raises_error(self): + """ Should return True if `stdin` is a socket and raises error. """ + test_fd = 23 + getsockopt = self.mock_socket.getsockopt + getsockopt.side_effect = socket.error( + object(), "Weird socket stuff") + expected_result = True + result = daemon.daemon.is_socket(test_fd) + self.assertIs(result, expected_result) + + +class is_process_started_by_superserver_TestCase(scaffold.TestCase): + """ Test cases for is_process_started_by_superserver function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(is_process_started_by_superserver_TestCase, self).setUp() + + def fake_is_socket(fd): + if sys.__stdin__.fileno() == fd: + result = self.fake_stdin_is_socket_func() + else: + result = False + return result + + self.fake_stdin_is_socket_func = (lambda: False) + + func_patcher_is_socket = mock.patch.object( + daemon.daemon, "is_socket", + side_effect=fake_is_socket) + func_patcher_is_socket.start() + self.addCleanup(func_patcher_is_socket.stop) + + def test_returns_false_by_default(self): + """ Should return False under normal circumstances. """ + expected_result = False + result = daemon.daemon.is_process_started_by_superserver() + self.assertIs(result, expected_result) + + def test_returns_true_if_stdin_is_socket(self): + """ Should return True if `stdin` is a socket. """ + self.fake_stdin_is_socket_func = (lambda: True) + expected_result = True + result = daemon.daemon.is_process_started_by_superserver() + self.assertIs(result, expected_result) + + +@mock.patch.object( + daemon.daemon, "is_process_started_by_superserver", + return_value=False) +@mock.patch.object( + daemon.daemon, "is_process_started_by_init", + return_value=False) +class is_detach_process_context_required_TestCase(scaffold.TestCase): + """ Test cases for is_detach_process_context_required function. """ + + def test_returns_true_by_default( + self, + mock_func_is_process_started_by_init, + mock_func_is_process_started_by_superserver): + """ Should return True under normal circumstances. """ + expected_result = True + result = daemon.daemon.is_detach_process_context_required() + self.assertIs(result, expected_result) + + def test_returns_false_if_started_by_init( + self, + mock_func_is_process_started_by_init, + mock_func_is_process_started_by_superserver): + """ Should return False if current process started by init. """ + mock_func_is_process_started_by_init.return_value = True + expected_result = False + result = daemon.daemon.is_detach_process_context_required() + self.assertIs(result, expected_result) + + def test_returns_true_if_started_by_superserver( + self, + mock_func_is_process_started_by_init, + mock_func_is_process_started_by_superserver): + """ Should return False if current process started by superserver. """ + mock_func_is_process_started_by_superserver.return_value = True + expected_result = False + result = daemon.daemon.is_detach_process_context_required() + self.assertIs(result, expected_result) + + +def setup_streams_fixtures(testcase): + """ Set up common test fixtures for standard streams. """ + testcase.stream_file_paths = dict( + stdin=tempfile.mktemp(), + stdout=tempfile.mktemp(), + stderr=tempfile.mktemp(), + ) + + testcase.stream_files_by_name = dict( + (name, FakeFileDescriptorStringIO()) + for name in ['stdin', 'stdout', 'stderr'] + ) + + testcase.stream_files_by_path = dict( + (testcase.stream_file_paths[name], + testcase.stream_files_by_name[name]) + for name in ['stdin', 'stdout', 'stderr'] + ) + + +@mock.patch.object(os, "dup2") +class redirect_stream_TestCase(scaffold.TestCase): + """ Test cases for redirect_stream function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(redirect_stream_TestCase, self).setUp() + + self.test_system_stream = FakeFileDescriptorStringIO() + self.test_target_stream = FakeFileDescriptorStringIO() + self.test_null_file = FakeFileDescriptorStringIO() + + def fake_os_open(path, flag, mode=None): + if path == os.devnull: + result = self.test_null_file.fileno() + else: + raise FileNotFoundError("No such file", path) + return result + + func_patcher_os_open = mock.patch.object( + os, "open", + side_effect=fake_os_open) + self.mock_func_os_open = func_patcher_os_open.start() + self.addCleanup(func_patcher_os_open.stop) + + def test_duplicates_target_file_descriptor( + self, mock_func_os_dup2): + """ Should duplicate file descriptor from target to system stream. """ + system_stream = self.test_system_stream + system_fileno = system_stream.fileno() + target_stream = self.test_target_stream + target_fileno = target_stream.fileno() + daemon.daemon.redirect_stream(system_stream, target_stream) + mock_func_os_dup2.assert_called_with(target_fileno, system_fileno) + + def test_duplicates_null_file_descriptor_by_default( + self, mock_func_os_dup2): + """ Should by default duplicate the null file to the system stream. """ + system_stream = self.test_system_stream + system_fileno = system_stream.fileno() + target_stream = None + null_path = os.devnull + null_flag = os.O_RDWR + null_file = self.test_null_file + null_fileno = null_file.fileno() + daemon.daemon.redirect_stream(system_stream, target_stream) + self.mock_func_os_open.assert_called_with(null_path, null_flag) + mock_func_os_dup2.assert_called_with(null_fileno, system_fileno) + + +class make_default_signal_map_TestCase(scaffold.TestCase): + """ Test cases for make_default_signal_map function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(make_default_signal_map_TestCase, self).setUp() + + # Use whatever default string type this Python version needs. + signal_module_name = str('signal') + self.fake_signal_module = ModuleType(signal_module_name) + + fake_signal_names = [ + 'SIGHUP', + 'SIGCLD', + 'SIGSEGV', + 'SIGTSTP', + 'SIGTTIN', + 'SIGTTOU', + 'SIGTERM', + ] + for name in fake_signal_names: + setattr(self.fake_signal_module, name, object()) + + module_patcher_signal = mock.patch.object( + daemon.daemon, "signal", new=self.fake_signal_module) + module_patcher_signal.start() + self.addCleanup(module_patcher_signal.stop) + + default_signal_map_by_name = { + 'SIGTSTP': None, + 'SIGTTIN': None, + 'SIGTTOU': None, + 'SIGTERM': 'terminate', + } + self.default_signal_map = dict( + (getattr(self.fake_signal_module, name), target) + for (name, target) in default_signal_map_by_name.items()) + + def test_returns_constructed_signal_map(self): + """ Should return map per default. """ + expected_result = self.default_signal_map + result = daemon.daemon.make_default_signal_map() + self.assertEqual(expected_result, result) + + def test_returns_signal_map_with_only_ids_in_signal_module(self): + """ Should return map with only signals in the `signal` module. + + The `signal` module is documented to only define those + signals which exist on the running system. Therefore the + default map should not contain any signals which are not + defined in the `signal` module. + + """ + del(self.default_signal_map[self.fake_signal_module.SIGTTOU]) + del(self.fake_signal_module.SIGTTOU) + expected_result = self.default_signal_map + result = daemon.daemon.make_default_signal_map() + self.assertEqual(expected_result, result) + + +@mock.patch.object(daemon.daemon.signal, "signal") +class set_signal_handlers_TestCase(scaffold.TestCase): + """ Test cases for set_signal_handlers function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(set_signal_handlers_TestCase, self).setUp() + + self.signal_handler_map = { + signal.SIGQUIT: object(), + signal.SIGSEGV: object(), + signal.SIGINT: object(), + } + + def test_sets_signal_handler_for_each_item(self, mock_func_signal_signal): + """ Should set signal handler for each item in map. """ + signal_handler_map = self.signal_handler_map + expected_calls = [ + mock.call(signal_number, handler) + for (signal_number, handler) in signal_handler_map.items()] + daemon.daemon.set_signal_handlers(signal_handler_map) + self.assertEquals(expected_calls, mock_func_signal_signal.mock_calls) + + +@mock.patch.object(daemon.daemon.atexit, "register") +class register_atexit_function_TestCase(scaffold.TestCase): + """ Test cases for register_atexit_function function. """ + + def test_registers_function_for_atexit_processing( + self, mock_func_atexit_register): + """ Should register specified function for atexit processing. """ + func = object() + daemon.daemon.register_atexit_function(func) + mock_func_atexit_register.assert_called_with(func) + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/test/test_metadata.py b/scripts/external_libs/python-daemon-2.0.5/test/test_metadata.py new file mode 100644 index 00000000..692753f4 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/test/test_metadata.py @@ -0,0 +1,380 @@ +# -*- coding: utf-8 -*- +# +# test/test_metadata.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Unit test for ‘_metadata’ private module. + """ + +from __future__ import (absolute_import, unicode_literals) + +import sys +import errno +import re +try: + # Python 3 standard library. + import urllib.parse as urlparse +except ImportError: + # Python 2 standard library. + import urlparse +import functools +import collections +import json + +import pkg_resources +import mock +import testtools.helpers +import testtools.matchers +import testscenarios + +from . import scaffold +from .scaffold import (basestring, unicode) + +import daemon._metadata as metadata + + +class HasAttribute(testtools.matchers.Matcher): + """ A matcher to assert an object has a named attribute. """ + + def __init__(self, name): + self.attribute_name = name + + def match(self, instance): + """ Assert the object `instance` has an attribute named `name`. """ + result = None + if not testtools.helpers.safe_hasattr(instance, self.attribute_name): + result = AttributeNotFoundMismatch(instance, self.attribute_name) + return result + + +class AttributeNotFoundMismatch(testtools.matchers.Mismatch): + """ The specified instance does not have the named attribute. """ + + def __init__(self, instance, name): + self.instance = instance + self.attribute_name = name + + def describe(self): + """ Emit a text description of this mismatch. """ + text = ( + "{instance!r}" + " has no attribute named {name!r}").format( + instance=self.instance, name=self.attribute_name) + return text + + +class metadata_value_TestCase(scaffold.TestCaseWithScenarios): + """ Test cases for metadata module values. """ + + expected_str_attributes = set([ + 'version_installed', + 'author', + 'copyright', + 'license', + 'url', + ]) + + scenarios = [ + (name, {'attribute_name': name}) + for name in expected_str_attributes] + for (name, params) in scenarios: + if name == 'version_installed': + # No duck typing, this attribute might be None. + params['ducktype_attribute_name'] = NotImplemented + continue + # Expect an attribute of ‘str’ to test this value. + params['ducktype_attribute_name'] = 'isdigit' + + def test_module_has_attribute(self): + """ Metadata should have expected value as a module attribute. """ + self.assertThat( + metadata, HasAttribute(self.attribute_name)) + + def test_module_attribute_has_duck_type(self): + """ Metadata value should have expected duck-typing attribute. """ + if self.ducktype_attribute_name == NotImplemented: + self.skipTest("Can't assert this attribute's type") + instance = getattr(metadata, self.attribute_name) + self.assertThat( + instance, HasAttribute(self.ducktype_attribute_name)) + + +class parse_person_field_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘get_latest_version’ function. """ + + scenarios = [ + ('simple', { + 'test_person': "Foo Bar <foo.bar@example.com>", + 'expected_result': ("Foo Bar", "foo.bar@example.com"), + }), + ('empty', { + 'test_person': "", + 'expected_result': (None, None), + }), + ('none', { + 'test_person': None, + 'expected_error': TypeError, + }), + ('no email', { + 'test_person': "Foo Bar", + 'expected_result': ("Foo Bar", None), + }), + ] + + def test_returns_expected_result(self): + """ Should return expected result. """ + if hasattr(self, 'expected_error'): + self.assertRaises( + self.expected_error, + metadata.parse_person_field, self.test_person) + else: + result = metadata.parse_person_field(self.test_person) + self.assertEqual(self.expected_result, result) + + +class YearRange_TestCase(scaffold.TestCaseWithScenarios): + """ Test cases for ‘YearRange’ class. """ + + scenarios = [ + ('simple', { + 'begin_year': 1970, + 'end_year': 1979, + 'expected_text': "1970–1979", + }), + ('same year', { + 'begin_year': 1970, + 'end_year': 1970, + 'expected_text': "1970", + }), + ('no end year', { + 'begin_year': 1970, + 'end_year': None, + 'expected_text': "1970", + }), + ] + + def setUp(self): + """ Set up test fixtures. """ + super(YearRange_TestCase, self).setUp() + + self.test_instance = metadata.YearRange( + self.begin_year, self.end_year) + + def test_text_representation_as_expected(self): + """ Text representation should be as expected. """ + result = unicode(self.test_instance) + self.assertEqual(result, self.expected_text) + + +FakeYearRange = collections.namedtuple('FakeYearRange', ['begin', 'end']) + +@mock.patch.object(metadata, 'YearRange', new=FakeYearRange) +class make_year_range_TestCase(scaffold.TestCaseWithScenarios): + """ Test cases for ‘make_year_range’ function. """ + + scenarios = [ + ('simple', { + 'begin_year': "1970", + 'end_date': "1979-01-01", + 'expected_range': FakeYearRange(begin=1970, end=1979), + }), + ('same year', { + 'begin_year': "1970", + 'end_date': "1970-01-01", + 'expected_range': FakeYearRange(begin=1970, end=1970), + }), + ('no end year', { + 'begin_year': "1970", + 'end_date': None, + 'expected_range': FakeYearRange(begin=1970, end=None), + }), + ('end date UNKNOWN token', { + 'begin_year': "1970", + 'end_date': "UNKNOWN", + 'expected_range': FakeYearRange(begin=1970, end=None), + }), + ('end date FUTURE token', { + 'begin_year': "1970", + 'end_date': "FUTURE", + 'expected_range': FakeYearRange(begin=1970, end=None), + }), + ] + + def test_result_matches_expected_range(self): + """ Result should match expected YearRange. """ + result = metadata.make_year_range(self.begin_year, self.end_date) + self.assertEqual(result, self.expected_range) + + +class metadata_content_TestCase(scaffold.TestCase): + """ Test cases for content of metadata. """ + + def test_copyright_formatted_correctly(self): + """ Copyright statement should be formatted correctly. """ + regex_pattern = ( + "Copyright © " + "\d{4}" # four-digit year + "(?:–\d{4})?" # optional range dash and ending four-digit year + ) + regex_flags = re.UNICODE + self.assertThat( + metadata.copyright, + testtools.matchers.MatchesRegex(regex_pattern, regex_flags)) + + def test_author_formatted_correctly(self): + """ Author information should be formatted correctly. """ + regex_pattern = ( + ".+ " # name + "<[^>]+>" # email address, in angle brackets + ) + regex_flags = re.UNICODE + self.assertThat( + metadata.author, + testtools.matchers.MatchesRegex(regex_pattern, regex_flags)) + + def test_copyright_contains_author(self): + """ Copyright information should contain author information. """ + self.assertThat( + metadata.copyright, + testtools.matchers.Contains(metadata.author)) + + def test_url_parses_correctly(self): + """ Homepage URL should parse correctly. """ + result = urlparse.urlparse(metadata.url) + self.assertIsInstance( + result, urlparse.ParseResult, + "URL value {url!r} did not parse correctly".format( + url=metadata.url)) + + +try: + FileNotFoundError +except NameError: + # Python 2 uses IOError. + FileNotFoundError = functools.partial(IOError, errno.ENOENT) + +version_info_filename = "version_info.json" + +def fake_func_has_metadata(testcase, resource_name): + """ Fake the behaviour of ‘pkg_resources.Distribution.has_metadata’. """ + if ( + resource_name != testcase.expected_resource_name + or not hasattr(testcase, 'test_version_info')): + return False + return True + + +def fake_func_get_metadata(testcase, resource_name): + """ Fake the behaviour of ‘pkg_resources.Distribution.get_metadata’. """ + if not fake_func_has_metadata(testcase, resource_name): + error = FileNotFoundError(resource_name) + raise error + content = testcase.test_version_info + return content + + +def fake_func_get_distribution(testcase, distribution_name): + """ Fake the behaviour of ‘pkg_resources.get_distribution’. """ + if distribution_name != metadata.distribution_name: + raise pkg_resources.DistributionNotFound + if hasattr(testcase, 'get_distribution_error'): + raise testcase.get_distribution_error + mock_distribution = testcase.mock_distribution + mock_distribution.has_metadata.side_effect = functools.partial( + fake_func_has_metadata, testcase) + mock_distribution.get_metadata.side_effect = functools.partial( + fake_func_get_metadata, testcase) + return mock_distribution + + +@mock.patch.object(metadata, 'distribution_name', new="mock-dist") +class get_distribution_version_info_TestCase(scaffold.TestCaseWithScenarios): + """ Test cases for ‘get_distribution_version_info’ function. """ + + default_version_info = { + 'release_date': "UNKNOWN", + 'version': "UNKNOWN", + 'maintainer': "UNKNOWN", + } + + scenarios = [ + ('version 0.0', { + 'test_version_info': json.dumps({ + 'version': "0.0", + }), + 'expected_version_info': {'version': "0.0"}, + }), + ('version 1.0', { + 'test_version_info': json.dumps({ + 'version': "1.0", + }), + 'expected_version_info': {'version': "1.0"}, + }), + ('file lorem_ipsum.json', { + 'version_info_filename': "lorem_ipsum.json", + 'test_version_info': json.dumps({ + 'version': "1.0", + }), + 'expected_version_info': {'version': "1.0"}, + }), + ('not installed', { + 'get_distribution_error': pkg_resources.DistributionNotFound(), + 'expected_version_info': default_version_info, + }), + ('no version_info', { + 'expected_version_info': default_version_info, + }), + ] + + def setUp(self): + """ Set up test fixtures. """ + super(get_distribution_version_info_TestCase, self).setUp() + + if hasattr(self, 'expected_resource_name'): + self.test_args = {'filename': self.expected_resource_name} + else: + self.test_args = {} + self.expected_resource_name = version_info_filename + + self.mock_distribution = mock.MagicMock() + func_patcher_get_distribution = mock.patch.object( + pkg_resources, 'get_distribution') + func_patcher_get_distribution.start() + self.addCleanup(func_patcher_get_distribution.stop) + pkg_resources.get_distribution.side_effect = functools.partial( + fake_func_get_distribution, self) + + def test_requests_installed_distribution(self): + """ The package distribution should be retrieved. """ + expected_distribution_name = metadata.distribution_name + version_info = metadata.get_distribution_version_info(**self.test_args) + pkg_resources.get_distribution.assert_called_with( + expected_distribution_name) + + def test_requests_specified_filename(self): + """ The specified metadata resource name should be requested. """ + if hasattr(self, 'get_distribution_error'): + self.skipTest("No access to distribution") + version_info = metadata.get_distribution_version_info(**self.test_args) + self.mock_distribution.has_metadata.assert_called_with( + self.expected_resource_name) + + def test_result_matches_expected_items(self): + """ The result should match the expected items. """ + version_info = metadata.get_distribution_version_info(**self.test_args) + self.assertEqual(self.expected_version_info, version_info) + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/test/test_pidfile.py b/scripts/external_libs/python-daemon-2.0.5/test/test_pidfile.py new file mode 100644 index 00000000..9b636ec8 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/test/test_pidfile.py @@ -0,0 +1,472 @@ +# -*- coding: utf-8 -*- +# +# test/test_pidfile.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Unit test for ‘pidfile’ module. + """ + +from __future__ import (absolute_import, unicode_literals) + +try: + # Python 3 standard library. + import builtins +except ImportError: + # Python 2 standard library. + import __builtin__ as builtins +import os +import itertools +import tempfile +import errno +import functools +try: + # Standard library of Python 2.7 and later. + from io import StringIO +except ImportError: + # Standard library of Python 2.6 and earlier. + from StringIO import StringIO + +import mock +import lockfile + +from . import scaffold + +import daemon.pidfile + + +class FakeFileDescriptorStringIO(StringIO, object): + """ A StringIO class that fakes a file descriptor. """ + + _fileno_generator = itertools.count() + + def __init__(self, *args, **kwargs): + self._fileno = next(self._fileno_generator) + super(FakeFileDescriptorStringIO, self).__init__(*args, **kwargs) + + def fileno(self): + return self._fileno + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + +try: + FileNotFoundError + PermissionError +except NameError: + # Python 2 uses IOError. + FileNotFoundError = functools.partial(IOError, errno.ENOENT) + PermissionError = functools.partial(IOError, errno.EPERM) + + +def make_pidlockfile_scenarios(): + """ Make a collection of scenarios for testing `PIDLockFile` instances. + + :return: A collection of scenarios for tests involving + `PIDLockfFile` instances. + + The collection is a mapping from scenario name to a dictionary of + scenario attributes. + + """ + + fake_current_pid = 235 + fake_other_pid = 8642 + fake_pidfile_path = tempfile.mktemp() + + fake_pidfile_empty = FakeFileDescriptorStringIO() + fake_pidfile_current_pid = FakeFileDescriptorStringIO( + "{pid:d}\n".format(pid=fake_current_pid)) + fake_pidfile_other_pid = FakeFileDescriptorStringIO( + "{pid:d}\n".format(pid=fake_other_pid)) + fake_pidfile_bogus = FakeFileDescriptorStringIO( + "b0gUs") + + scenarios = { + 'simple': {}, + 'not-exist': { + 'open_func_name': 'fake_open_nonexist', + 'os_open_func_name': 'fake_os_open_nonexist', + }, + 'not-exist-write-denied': { + 'open_func_name': 'fake_open_nonexist', + 'os_open_func_name': 'fake_os_open_nonexist', + }, + 'not-exist-write-busy': { + 'open_func_name': 'fake_open_nonexist', + 'os_open_func_name': 'fake_os_open_nonexist', + }, + 'exist-read-denied': { + 'open_func_name': 'fake_open_read_denied', + 'os_open_func_name': 'fake_os_open_read_denied', + }, + 'exist-locked-read-denied': { + 'locking_pid': fake_other_pid, + 'open_func_name': 'fake_open_read_denied', + 'os_open_func_name': 'fake_os_open_read_denied', + }, + 'exist-empty': {}, + 'exist-invalid': { + 'pidfile': fake_pidfile_bogus, + }, + 'exist-current-pid': { + 'pidfile': fake_pidfile_current_pid, + 'pidfile_pid': fake_current_pid, + }, + 'exist-current-pid-locked': { + 'pidfile': fake_pidfile_current_pid, + 'pidfile_pid': fake_current_pid, + 'locking_pid': fake_current_pid, + }, + 'exist-other-pid': { + 'pidfile': fake_pidfile_other_pid, + 'pidfile_pid': fake_other_pid, + }, + 'exist-other-pid-locked': { + 'pidfile': fake_pidfile_other_pid, + 'pidfile_pid': fake_other_pid, + 'locking_pid': fake_other_pid, + }, + } + + for scenario in scenarios.values(): + scenario['pid'] = fake_current_pid + scenario['pidfile_path'] = fake_pidfile_path + if 'pidfile' not in scenario: + scenario['pidfile'] = fake_pidfile_empty + if 'pidfile_pid' not in scenario: + scenario['pidfile_pid'] = None + if 'locking_pid' not in scenario: + scenario['locking_pid'] = None + if 'open_func_name' not in scenario: + scenario['open_func_name'] = 'fake_open_okay' + if 'os_open_func_name' not in scenario: + scenario['os_open_func_name'] = 'fake_os_open_okay' + + return scenarios + + +def setup_pidfile_fixtures(testcase): + """ Set up common fixtures for PID file test cases. + + :param testcase: A `TestCase` instance to decorate. + + Decorate the `testcase` with attributes to be fixtures for tests + involving `PIDLockFile` instances. + + """ + scenarios = make_pidlockfile_scenarios() + testcase.pidlockfile_scenarios = scenarios + + def get_scenario_option(testcase, key, default=None): + value = default + try: + value = testcase.scenario[key] + except (NameError, TypeError, AttributeError, KeyError): + pass + return value + + func_patcher_os_getpid = mock.patch.object( + os, "getpid", + return_value=scenarios['simple']['pid']) + func_patcher_os_getpid.start() + testcase.addCleanup(func_patcher_os_getpid.stop) + + def make_fake_open_funcs(testcase): + + def fake_open_nonexist(filename, mode, buffering): + if mode.startswith('r'): + error = FileNotFoundError( + "No such file {filename!r}".format( + filename=filename)) + raise error + else: + result = testcase.scenario['pidfile'] + return result + + def fake_open_read_denied(filename, mode, buffering): + if mode.startswith('r'): + error = PermissionError( + "Read denied on {filename!r}".format( + filename=filename)) + raise error + else: + result = testcase.scenario['pidfile'] + return result + + def fake_open_okay(filename, mode, buffering): + result = testcase.scenario['pidfile'] + return result + + def fake_os_open_nonexist(filename, flags, mode): + if (flags & os.O_CREAT): + result = testcase.scenario['pidfile'].fileno() + else: + error = FileNotFoundError( + "No such file {filename!r}".format( + filename=filename)) + raise error + return result + + def fake_os_open_read_denied(filename, flags, mode): + if (flags & os.O_CREAT): + result = testcase.scenario['pidfile'].fileno() + else: + error = PermissionError( + "Read denied on {filename!r}".format( + filename=filename)) + raise error + return result + + def fake_os_open_okay(filename, flags, mode): + result = testcase.scenario['pidfile'].fileno() + return result + + funcs = dict( + (name, obj) for (name, obj) in vars().items() + if callable(obj)) + + return funcs + + testcase.fake_pidfile_open_funcs = make_fake_open_funcs(testcase) + + def fake_open(filename, mode='rt', buffering=None): + scenario_path = get_scenario_option(testcase, 'pidfile_path') + if filename == scenario_path: + func_name = testcase.scenario['open_func_name'] + fake_open_func = testcase.fake_pidfile_open_funcs[func_name] + result = fake_open_func(filename, mode, buffering) + else: + result = FakeFileDescriptorStringIO() + return result + + mock_open = mock.mock_open() + mock_open.side_effect = fake_open + + func_patcher_builtin_open = mock.patch.object( + builtins, "open", + new=mock_open) + func_patcher_builtin_open.start() + testcase.addCleanup(func_patcher_builtin_open.stop) + + def fake_os_open(filename, flags, mode=None): + scenario_path = get_scenario_option(testcase, 'pidfile_path') + if filename == scenario_path: + func_name = testcase.scenario['os_open_func_name'] + fake_os_open_func = testcase.fake_pidfile_open_funcs[func_name] + result = fake_os_open_func(filename, flags, mode) + else: + result = FakeFileDescriptorStringIO().fileno() + return result + + mock_os_open = mock.MagicMock(side_effect=fake_os_open) + + func_patcher_os_open = mock.patch.object( + os, "open", + new=mock_os_open) + func_patcher_os_open.start() + testcase.addCleanup(func_patcher_os_open.stop) + + def fake_os_fdopen(fd, mode='rt', buffering=None): + scenario_pidfile = get_scenario_option( + testcase, 'pidfile', FakeFileDescriptorStringIO()) + if fd == testcase.scenario['pidfile'].fileno(): + result = testcase.scenario['pidfile'] + else: + raise OSError(errno.EBADF, "Bad file descriptor") + return result + + mock_os_fdopen = mock.MagicMock(side_effect=fake_os_fdopen) + + func_patcher_os_fdopen = mock.patch.object( + os, "fdopen", + new=mock_os_fdopen) + func_patcher_os_fdopen.start() + testcase.addCleanup(func_patcher_os_fdopen.stop) + + +def make_lockfile_method_fakes(scenario): + """ Make common fake methods for lockfile class. + + :param scenario: A scenario for testing with PIDLockFile. + :return: A mapping from normal function name to the corresponding + fake function. + + Each fake function behaves appropriately for the specified `scenario`. + + """ + + def fake_func_read_pid(): + return scenario['pidfile_pid'] + def fake_func_is_locked(): + return (scenario['locking_pid'] is not None) + def fake_func_i_am_locking(): + return ( + scenario['locking_pid'] == scenario['pid']) + def fake_func_acquire(timeout=None): + if scenario['locking_pid'] is not None: + raise lockfile.AlreadyLocked() + scenario['locking_pid'] = scenario['pid'] + def fake_func_release(): + if scenario['locking_pid'] is None: + raise lockfile.NotLocked() + if scenario['locking_pid'] != scenario['pid']: + raise lockfile.NotMyLock() + scenario['locking_pid'] = None + def fake_func_break_lock(): + scenario['locking_pid'] = None + + fake_methods = dict( + ( + func_name.replace('fake_func_', ''), + mock.MagicMock(side_effect=fake_func)) + for (func_name, fake_func) in vars().items() + if func_name.startswith('fake_func_')) + + return fake_methods + + +def apply_lockfile_method_mocks(mock_lockfile, testcase, scenario): + """ Apply common fake methods to mock lockfile class. + + :param mock_lockfile: An object providing the `LockFile` interface. + :param testcase: The `TestCase` instance providing the context for + the patch. + :param scenario: The `PIDLockFile` test scenario to use. + + Mock the `LockFile` methods of `mock_lockfile`, by applying fake + methods customised for `scenario`. The mock is does by a patch + within the context of `testcase`. + + """ + fake_methods = dict( + (func_name, fake_func) + for (func_name, fake_func) in + make_lockfile_method_fakes(scenario).items() + if func_name not in ['read_pid']) + + for (func_name, fake_func) in fake_methods.items(): + func_patcher = mock.patch.object( + mock_lockfile, func_name, + new=fake_func) + func_patcher.start() + testcase.addCleanup(func_patcher.stop) + + +def setup_pidlockfile_fixtures(testcase, scenario_name=None): + """ Set up common fixtures for PIDLockFile test cases. + + :param testcase: The `TestCase` instance to decorate. + :param scenario_name: The name of the `PIDLockFile` scenario to use. + + Decorate the `testcase` with attributes that are fixtures for test + cases involving `PIDLockFile` instances.` + + """ + + setup_pidfile_fixtures(testcase) + + for func_name in [ + 'write_pid_to_pidfile', + 'remove_existing_pidfile', + ]: + func_patcher = mock.patch.object(lockfile.pidlockfile, func_name) + func_patcher.start() + testcase.addCleanup(func_patcher.stop) + + +class TimeoutPIDLockFile_TestCase(scaffold.TestCase): + """ Test cases for ‘TimeoutPIDLockFile’ class. """ + + def setUp(self): + """ Set up test fixtures. """ + super(TimeoutPIDLockFile_TestCase, self).setUp() + + pidlockfile_scenarios = make_pidlockfile_scenarios() + self.pidlockfile_scenario = pidlockfile_scenarios['simple'] + pidfile_path = self.pidlockfile_scenario['pidfile_path'] + + for func_name in ['__init__', 'acquire']: + func_patcher = mock.patch.object( + lockfile.pidlockfile.PIDLockFile, func_name) + func_patcher.start() + self.addCleanup(func_patcher.stop) + + self.scenario = { + 'pidfile_path': self.pidlockfile_scenario['pidfile_path'], + 'acquire_timeout': self.getUniqueInteger(), + } + + self.test_kwargs = dict( + path=self.scenario['pidfile_path'], + acquire_timeout=self.scenario['acquire_timeout'], + ) + self.test_instance = daemon.pidfile.TimeoutPIDLockFile( + **self.test_kwargs) + + def test_inherits_from_pidlockfile(self): + """ Should inherit from PIDLockFile. """ + instance = self.test_instance + self.assertIsInstance(instance, lockfile.pidlockfile.PIDLockFile) + + def test_init_has_expected_signature(self): + """ Should have expected signature for ‘__init__’. """ + def test_func(self, path, acquire_timeout=None, *args, **kwargs): pass + test_func.__name__ = str('__init__') + self.assertFunctionSignatureMatch( + test_func, + daemon.pidfile.TimeoutPIDLockFile.__init__) + + def test_has_specified_acquire_timeout(self): + """ Should have specified ‘acquire_timeout’ value. """ + instance = self.test_instance + expected_timeout = self.test_kwargs['acquire_timeout'] + self.assertEqual(expected_timeout, instance.acquire_timeout) + + @mock.patch.object( + lockfile.pidlockfile.PIDLockFile, "__init__", + autospec=True) + def test_calls_superclass_init(self, mock_init): + """ Should call the superclass ‘__init__’. """ + expected_path = self.test_kwargs['path'] + instance = daemon.pidfile.TimeoutPIDLockFile(**self.test_kwargs) + mock_init.assert_called_with(instance, expected_path) + + @mock.patch.object( + lockfile.pidlockfile.PIDLockFile, "acquire", + autospec=True) + def test_acquire_uses_specified_timeout(self, mock_func_acquire): + """ Should call the superclass ‘acquire’ with specified timeout. """ + instance = self.test_instance + test_timeout = self.getUniqueInteger() + expected_timeout = test_timeout + instance.acquire(test_timeout) + mock_func_acquire.assert_called_with(instance, expected_timeout) + + @mock.patch.object( + lockfile.pidlockfile.PIDLockFile, "acquire", + autospec=True) + def test_acquire_uses_stored_timeout_by_default(self, mock_func_acquire): + """ Should call superclass ‘acquire’ with stored timeout by default. """ + instance = self.test_instance + test_timeout = self.test_kwargs['acquire_timeout'] + expected_timeout = test_timeout + instance.acquire() + mock_func_acquire.assert_called_with(instance, expected_timeout) + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/test/test_runner.py b/scripts/external_libs/python-daemon-2.0.5/test/test_runner.py new file mode 100644 index 00000000..4c0c714a --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/test/test_runner.py @@ -0,0 +1,675 @@ +# -*- coding: utf-8 -*- +# +# test/test_runner.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2009–2015 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the Apache License, version 2.0 as published by the +# Apache Software Foundation. +# No warranty expressed or implied. See the file ‘LICENSE.ASF-2’ for details. + +""" Unit test for ‘runner’ module. + """ + +from __future__ import (absolute_import, unicode_literals) + +try: + # Python 3 standard library. + import builtins +except ImportError: + # Python 2 standard library. + import __builtin__ as builtins +import os +import os.path +import sys +import tempfile +import errno +import signal +import functools + +import lockfile +import mock +import testtools + +from . import scaffold +from .scaffold import (basestring, unicode) +from .test_pidfile import ( + FakeFileDescriptorStringIO, + setup_pidfile_fixtures, + make_pidlockfile_scenarios, + apply_lockfile_method_mocks, + ) +from .test_daemon import ( + setup_streams_fixtures, + ) + +import daemon.daemon +import daemon.runner +import daemon.pidfile + + +class ModuleExceptions_TestCase(scaffold.Exception_TestCase): + """ Test cases for module exception classes. """ + + scenarios = scaffold.make_exception_scenarios([ + ('daemon.runner.DaemonRunnerError', dict( + exc_type = daemon.runner.DaemonRunnerError, + min_args = 1, + types = [Exception], + )), + ('daemon.runner.DaemonRunnerInvalidActionError', dict( + exc_type = daemon.runner.DaemonRunnerInvalidActionError, + min_args = 1, + types = [daemon.runner.DaemonRunnerError, ValueError], + )), + ('daemon.runner.DaemonRunnerStartFailureError', dict( + exc_type = daemon.runner.DaemonRunnerStartFailureError, + min_args = 1, + types = [daemon.runner.DaemonRunnerError, RuntimeError], + )), + ('daemon.runner.DaemonRunnerStopFailureError', dict( + exc_type = daemon.runner.DaemonRunnerStopFailureError, + min_args = 1, + types = [daemon.runner.DaemonRunnerError, RuntimeError], + )), + ]) + + +def make_runner_scenarios(): + """ Make a collection of scenarios for testing `DaemonRunner` instances. + + :return: A collection of scenarios for tests involving + `DaemonRunner` instances. + + The collection is a mapping from scenario name to a dictionary of + scenario attributes. + + """ + + pidlockfile_scenarios = make_pidlockfile_scenarios() + + scenarios = { + 'simple': { + 'pidlockfile_scenario_name': 'simple', + }, + 'pidfile-locked': { + 'pidlockfile_scenario_name': 'exist-other-pid-locked', + }, + } + + for scenario in scenarios.values(): + if 'pidlockfile_scenario_name' in scenario: + pidlockfile_scenario = pidlockfile_scenarios.pop( + scenario['pidlockfile_scenario_name']) + scenario['pid'] = pidlockfile_scenario['pid'] + scenario['pidfile_path'] = pidlockfile_scenario['pidfile_path'] + scenario['pidfile_timeout'] = 23 + scenario['pidlockfile_scenario'] = pidlockfile_scenario + + return scenarios + + +def set_runner_scenario(testcase, scenario_name): + """ Set the DaemonRunner test scenario for the test case. + + :param testcase: The `TestCase` instance to decorate. + :param scenario_name: The name of the scenario to use. + + Set the `DaemonRunner` test scenario name and decorate the + `testcase` with the corresponding scenario fixtures. + + """ + scenarios = testcase.runner_scenarios + testcase.scenario = scenarios[scenario_name] + apply_lockfile_method_mocks( + testcase.mock_runner_lockfile, + testcase, + testcase.scenario['pidlockfile_scenario']) + + +def setup_runner_fixtures(testcase): + """ Set up common fixtures for `DaemonRunner` test cases. + + :param testcase: A `TestCase` instance to decorate. + + Decorate the `testcase` with attributes to be fixtures for tests + involving `DaemonRunner` instances. + + """ + setup_pidfile_fixtures(testcase) + setup_streams_fixtures(testcase) + + testcase.runner_scenarios = make_runner_scenarios() + + patcher_stderr = mock.patch.object( + sys, "stderr", + new=FakeFileDescriptorStringIO()) + testcase.fake_stderr = patcher_stderr.start() + testcase.addCleanup(patcher_stderr.stop) + + simple_scenario = testcase.runner_scenarios['simple'] + + testcase.mock_runner_lockfile = mock.MagicMock( + spec=daemon.pidfile.TimeoutPIDLockFile) + apply_lockfile_method_mocks( + testcase.mock_runner_lockfile, + testcase, + simple_scenario['pidlockfile_scenario']) + testcase.mock_runner_lockfile.path = simple_scenario['pidfile_path'] + + patcher_lockfile_class = mock.patch.object( + daemon.pidfile, "TimeoutPIDLockFile", + return_value=testcase.mock_runner_lockfile) + patcher_lockfile_class.start() + testcase.addCleanup(patcher_lockfile_class.stop) + + class TestApp(object): + + def __init__(self): + self.stdin_path = testcase.stream_file_paths['stdin'] + self.stdout_path = testcase.stream_file_paths['stdout'] + self.stderr_path = testcase.stream_file_paths['stderr'] + self.pidfile_path = simple_scenario['pidfile_path'] + self.pidfile_timeout = simple_scenario['pidfile_timeout'] + + run = mock.MagicMock(name="TestApp.run") + + testcase.TestApp = TestApp + + patcher_runner_daemoncontext = mock.patch.object( + daemon.runner, "DaemonContext", autospec=True) + patcher_runner_daemoncontext.start() + testcase.addCleanup(patcher_runner_daemoncontext.stop) + + testcase.test_app = testcase.TestApp() + + testcase.test_program_name = "bazprog" + testcase.test_program_path = os.path.join( + "/foo/bar", testcase.test_program_name) + testcase.valid_argv_params = { + 'start': [testcase.test_program_path, 'start'], + 'stop': [testcase.test_program_path, 'stop'], + 'restart': [testcase.test_program_path, 'restart'], + } + + def fake_open(filename, mode=None, buffering=None): + if filename in testcase.stream_files_by_path: + result = testcase.stream_files_by_path[filename] + else: + result = FakeFileDescriptorStringIO() + result.mode = mode + result.buffering = buffering + return result + + mock_open = mock.mock_open() + mock_open.side_effect = fake_open + + func_patcher_builtin_open = mock.patch.object( + builtins, "open", + new=mock_open) + func_patcher_builtin_open.start() + testcase.addCleanup(func_patcher_builtin_open.stop) + + func_patcher_os_kill = mock.patch.object(os, "kill") + func_patcher_os_kill.start() + testcase.addCleanup(func_patcher_os_kill.stop) + + patcher_sys_argv = mock.patch.object( + sys, "argv", + new=testcase.valid_argv_params['start']) + patcher_sys_argv.start() + testcase.addCleanup(patcher_sys_argv.stop) + + testcase.test_instance = daemon.runner.DaemonRunner(testcase.test_app) + + testcase.scenario = NotImplemented + + +class DaemonRunner_BaseTestCase(scaffold.TestCase): + """ Base class for DaemonRunner test case classes. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonRunner_BaseTestCase, self).setUp() + + setup_runner_fixtures(self) + set_runner_scenario(self, 'simple') + + +class DaemonRunner_TestCase(DaemonRunner_BaseTestCase): + """ Test cases for DaemonRunner class. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonRunner_TestCase, self).setUp() + + func_patcher_parse_args = mock.patch.object( + daemon.runner.DaemonRunner, "parse_args") + func_patcher_parse_args.start() + self.addCleanup(func_patcher_parse_args.stop) + + # Create a new instance now with our custom patches. + self.test_instance = daemon.runner.DaemonRunner(self.test_app) + + def test_instantiate(self): + """ New instance of DaemonRunner should be created. """ + self.assertIsInstance(self.test_instance, daemon.runner.DaemonRunner) + + def test_parses_commandline_args(self): + """ Should parse commandline arguments. """ + self.test_instance.parse_args.assert_called_with() + + def test_has_specified_app(self): + """ Should have specified application object. """ + self.assertIs(self.test_app, self.test_instance.app) + + def test_sets_pidfile_none_when_pidfile_path_is_none(self): + """ Should set ‘pidfile’ to ‘None’ when ‘pidfile_path’ is ‘None’. """ + pidfile_path = None + self.test_app.pidfile_path = pidfile_path + expected_pidfile = None + instance = daemon.runner.DaemonRunner(self.test_app) + self.assertIs(expected_pidfile, instance.pidfile) + + def test_error_when_pidfile_path_not_string(self): + """ Should raise ValueError when PID file path not a string. """ + pidfile_path = object() + self.test_app.pidfile_path = pidfile_path + expected_error = ValueError + self.assertRaises( + expected_error, + daemon.runner.DaemonRunner, self.test_app) + + def test_error_when_pidfile_path_not_absolute(self): + """ Should raise ValueError when PID file path not absolute. """ + pidfile_path = "foo/bar.pid" + self.test_app.pidfile_path = pidfile_path + expected_error = ValueError + self.assertRaises( + expected_error, + daemon.runner.DaemonRunner, self.test_app) + + def test_creates_lock_with_specified_parameters(self): + """ Should create a TimeoutPIDLockFile with specified params. """ + pidfile_path = self.scenario['pidfile_path'] + pidfile_timeout = self.scenario['pidfile_timeout'] + daemon.pidfile.TimeoutPIDLockFile.assert_called_with( + pidfile_path, pidfile_timeout) + + def test_has_created_pidfile(self): + """ Should have new PID lock file as `pidfile` attribute. """ + expected_pidfile = self.mock_runner_lockfile + instance = self.test_instance + self.assertIs( + expected_pidfile, instance.pidfile) + + def test_daemon_context_has_created_pidfile(self): + """ DaemonContext component should have new PID lock file. """ + expected_pidfile = self.mock_runner_lockfile + daemon_context = self.test_instance.daemon_context + self.assertIs( + expected_pidfile, daemon_context.pidfile) + + def test_daemon_context_has_specified_stdin_stream(self): + """ DaemonContext component should have specified stdin file. """ + test_app = self.test_app + expected_file = self.stream_files_by_name['stdin'] + daemon_context = self.test_instance.daemon_context + self.assertEqual(expected_file, daemon_context.stdin) + + def test_daemon_context_has_stdin_in_read_mode(self): + """ DaemonContext component should open stdin file for read. """ + expected_mode = 'rt' + daemon_context = self.test_instance.daemon_context + self.assertIn(expected_mode, daemon_context.stdin.mode) + + def test_daemon_context_has_specified_stdout_stream(self): + """ DaemonContext component should have specified stdout file. """ + test_app = self.test_app + expected_file = self.stream_files_by_name['stdout'] + daemon_context = self.test_instance.daemon_context + self.assertEqual(expected_file, daemon_context.stdout) + + def test_daemon_context_has_stdout_in_append_mode(self): + """ DaemonContext component should open stdout file for append. """ + expected_mode = 'w+t' + daemon_context = self.test_instance.daemon_context + self.assertIn(expected_mode, daemon_context.stdout.mode) + + def test_daemon_context_has_specified_stderr_stream(self): + """ DaemonContext component should have specified stderr file. """ + test_app = self.test_app + expected_file = self.stream_files_by_name['stderr'] + daemon_context = self.test_instance.daemon_context + self.assertEqual(expected_file, daemon_context.stderr) + + def test_daemon_context_has_stderr_in_append_mode(self): + """ DaemonContext component should open stderr file for append. """ + expected_mode = 'w+t' + daemon_context = self.test_instance.daemon_context + self.assertIn(expected_mode, daemon_context.stderr.mode) + + def test_daemon_context_has_stderr_with_no_buffering(self): + """ DaemonContext component should open stderr file unbuffered. """ + expected_buffering = 0 + daemon_context = self.test_instance.daemon_context + self.assertEqual( + expected_buffering, daemon_context.stderr.buffering) + + +class DaemonRunner_usage_exit_TestCase(DaemonRunner_BaseTestCase): + """ Test cases for DaemonRunner.usage_exit method. """ + + def test_raises_system_exit(self): + """ Should raise SystemExit exception. """ + instance = self.test_instance + argv = [self.test_program_path] + self.assertRaises( + SystemExit, + instance._usage_exit, argv) + + def test_message_follows_conventional_format(self): + """ Should emit a conventional usage message. """ + instance = self.test_instance + argv = [self.test_program_path] + expected_stderr_output = """\ + usage: {progname} ... + """.format( + progname=self.test_program_name) + self.assertRaises( + SystemExit, + instance._usage_exit, argv) + self.assertOutputCheckerMatch( + expected_stderr_output, self.fake_stderr.getvalue()) + + +class DaemonRunner_parse_args_TestCase(DaemonRunner_BaseTestCase): + """ Test cases for DaemonRunner.parse_args method. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonRunner_parse_args_TestCase, self).setUp() + + func_patcher_usage_exit = mock.patch.object( + daemon.runner.DaemonRunner, "_usage_exit", + side_effect=NotImplementedError) + func_patcher_usage_exit.start() + self.addCleanup(func_patcher_usage_exit.stop) + + def test_emits_usage_message_if_insufficient_args(self): + """ Should emit a usage message and exit if too few arguments. """ + instance = self.test_instance + argv = [self.test_program_path] + exc = self.assertRaises( + NotImplementedError, + instance.parse_args, argv) + daemon.runner.DaemonRunner._usage_exit.assert_called_with(argv) + + def test_emits_usage_message_if_unknown_action_arg(self): + """ Should emit a usage message and exit if unknown action. """ + instance = self.test_instance + progname = self.test_program_name + argv = [self.test_program_path, 'bogus'] + exc = self.assertRaises( + NotImplementedError, + instance.parse_args, argv) + daemon.runner.DaemonRunner._usage_exit.assert_called_with(argv) + + def test_should_parse_system_argv_by_default(self): + """ Should parse sys.argv by default. """ + instance = self.test_instance + expected_action = 'start' + argv = self.valid_argv_params['start'] + with mock.patch.object(sys, "argv", new=argv): + instance.parse_args() + self.assertEqual(expected_action, instance.action) + + def test_sets_action_from_first_argument(self): + """ Should set action from first commandline argument. """ + instance = self.test_instance + for name, argv in self.valid_argv_params.items(): + expected_action = name + instance.parse_args(argv) + self.assertEqual(expected_action, instance.action) + + +try: + ProcessLookupError +except NameError: + # Python 2 uses OSError. + ProcessLookupError = functools.partial(OSError, errno.ESRCH) + +class DaemonRunner_do_action_TestCase(DaemonRunner_BaseTestCase): + """ Test cases for DaemonRunner.do_action method. """ + + def test_raises_error_if_unknown_action(self): + """ Should emit a usage message and exit if action is unknown. """ + instance = self.test_instance + instance.action = 'bogus' + expected_error = daemon.runner.DaemonRunnerInvalidActionError + self.assertRaises( + expected_error, + instance.do_action) + + +class DaemonRunner_do_action_start_TestCase(DaemonRunner_BaseTestCase): + """ Test cases for DaemonRunner.do_action method, action 'start'. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonRunner_do_action_start_TestCase, self).setUp() + + self.test_instance.action = 'start' + + def test_raises_error_if_pidfile_locked(self): + """ Should raise error if PID file is locked. """ + + instance = self.test_instance + instance.daemon_context.open.side_effect = lockfile.AlreadyLocked + pidfile_path = self.scenario['pidfile_path'] + expected_error = daemon.runner.DaemonRunnerStartFailureError + expected_message_content = pidfile_path + exc = self.assertRaises( + expected_error, + instance.do_action) + self.assertIn(expected_message_content, unicode(exc)) + + def test_breaks_lock_if_no_such_process(self): + """ Should request breaking lock if PID file process is not running. """ + set_runner_scenario(self, 'pidfile-locked') + instance = self.test_instance + self.mock_runner_lockfile.read_pid.return_value = ( + self.scenario['pidlockfile_scenario']['pidfile_pid']) + pidfile_path = self.scenario['pidfile_path'] + test_pid = self.scenario['pidlockfile_scenario']['pidfile_pid'] + expected_signal = signal.SIG_DFL + test_error = ProcessLookupError("Not running") + os.kill.side_effect = test_error + instance.do_action() + os.kill.assert_called_with(test_pid, expected_signal) + self.mock_runner_lockfile.break_lock.assert_called_with() + + def test_requests_daemon_context_open(self): + """ Should request the daemon context to open. """ + instance = self.test_instance + instance.do_action() + instance.daemon_context.open.assert_called_with() + + def test_emits_start_message_to_stderr(self): + """ Should emit start message to stderr. """ + instance = self.test_instance + expected_stderr = """\ + started with pid {pid:d} + """.format( + pid=self.scenario['pid']) + instance.do_action() + self.assertOutputCheckerMatch( + expected_stderr, self.fake_stderr.getvalue()) + + def test_requests_app_run(self): + """ Should request the application to run. """ + instance = self.test_instance + instance.do_action() + self.test_app.run.assert_called_with() + + +class DaemonRunner_do_action_stop_TestCase(DaemonRunner_BaseTestCase): + """ Test cases for DaemonRunner.do_action method, action 'stop'. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonRunner_do_action_stop_TestCase, self).setUp() + + set_runner_scenario(self, 'pidfile-locked') + + self.test_instance.action = 'stop' + + self.mock_runner_lockfile.is_locked.return_value = True + self.mock_runner_lockfile.i_am_locking.return_value = False + self.mock_runner_lockfile.read_pid.return_value = ( + self.scenario['pidlockfile_scenario']['pidfile_pid']) + + def test_raises_error_if_pidfile_not_locked(self): + """ Should raise error if PID file is not locked. """ + set_runner_scenario(self, 'simple') + instance = self.test_instance + self.mock_runner_lockfile.is_locked.return_value = False + self.mock_runner_lockfile.i_am_locking.return_value = False + self.mock_runner_lockfile.read_pid.return_value = ( + self.scenario['pidlockfile_scenario']['pidfile_pid']) + pidfile_path = self.scenario['pidfile_path'] + expected_error = daemon.runner.DaemonRunnerStopFailureError + expected_message_content = pidfile_path + exc = self.assertRaises( + expected_error, + instance.do_action) + self.assertIn(expected_message_content, unicode(exc)) + + def test_breaks_lock_if_pidfile_stale(self): + """ Should break lock if PID file is stale. """ + instance = self.test_instance + pidfile_path = self.scenario['pidfile_path'] + test_pid = self.scenario['pidlockfile_scenario']['pidfile_pid'] + expected_signal = signal.SIG_DFL + test_error = OSError(errno.ESRCH, "Not running") + os.kill.side_effect = test_error + instance.do_action() + self.mock_runner_lockfile.break_lock.assert_called_with() + + def test_sends_terminate_signal_to_process_from_pidfile(self): + """ Should send SIGTERM to the daemon process. """ + instance = self.test_instance + test_pid = self.scenario['pidlockfile_scenario']['pidfile_pid'] + expected_signal = signal.SIGTERM + instance.do_action() + os.kill.assert_called_with(test_pid, expected_signal) + + def test_raises_error_if_cannot_send_signal_to_process(self): + """ Should raise error if cannot send signal to daemon process. """ + instance = self.test_instance + test_pid = self.scenario['pidlockfile_scenario']['pidfile_pid'] + pidfile_path = self.scenario['pidfile_path'] + test_error = OSError(errno.EPERM, "Nice try") + os.kill.side_effect = test_error + expected_error = daemon.runner.DaemonRunnerStopFailureError + expected_message_content = unicode(test_pid) + exc = self.assertRaises( + expected_error, + instance.do_action) + self.assertIn(expected_message_content, unicode(exc)) + + +@mock.patch.object(daemon.runner.DaemonRunner, "_start") +@mock.patch.object(daemon.runner.DaemonRunner, "_stop") +class DaemonRunner_do_action_restart_TestCase(DaemonRunner_BaseTestCase): + """ Test cases for DaemonRunner.do_action method, action 'restart'. """ + + def setUp(self): + """ Set up test fixtures. """ + super(DaemonRunner_do_action_restart_TestCase, self).setUp() + + set_runner_scenario(self, 'pidfile-locked') + + self.test_instance.action = 'restart' + + def test_requests_stop_then_start( + self, + mock_func_daemonrunner_start, mock_func_daemonrunner_stop): + """ Should request stop, then start. """ + instance = self.test_instance + instance.do_action() + mock_func_daemonrunner_start.assert_called_with() + mock_func_daemonrunner_stop.assert_called_with() + + +@mock.patch.object(sys, "stderr") +class emit_message_TestCase(scaffold.TestCase): + """ Test cases for ‘emit_message’ function. """ + + def test_writes_specified_message_to_stream(self, mock_stderr): + """ Should write specified message to stream. """ + test_message = self.getUniqueString() + expected_content = "{message}\n".format(message=test_message) + daemon.runner.emit_message(test_message, stream=mock_stderr) + mock_stderr.write.assert_called_with(expected_content) + + def test_writes_to_specified_stream(self, mock_stderr): + """ Should write message to specified stream. """ + test_message = self.getUniqueString() + mock_stream = mock.MagicMock() + daemon.runner.emit_message(test_message, stream=mock_stream) + mock_stream.write.assert_called_with(mock.ANY) + + def test_writes_to_stderr_by_default(self, mock_stderr): + """ Should write message to ‘sys.stderr’ by default. """ + test_message = self.getUniqueString() + daemon.runner.emit_message(test_message) + mock_stderr.write.assert_called_with(mock.ANY) + + +class is_pidfile_stale_TestCase(scaffold.TestCase): + """ Test cases for ‘is_pidfile_stale’ function. """ + + def setUp(self): + """ Set up test fixtures. """ + super(is_pidfile_stale_TestCase, self).setUp() + + func_patcher_os_kill = mock.patch.object(os, "kill") + func_patcher_os_kill.start() + self.addCleanup(func_patcher_os_kill.stop) + os.kill.return_value = None + + self.test_pid = self.getUniqueInteger() + self.test_pidfile = mock.MagicMock(daemon.pidfile.TimeoutPIDLockFile) + self.test_pidfile.read_pid.return_value = self.test_pid + + def test_returns_false_if_no_pid_in_file(self): + """ Should return False if the pidfile contains no PID. """ + self.test_pidfile.read_pid.return_value = None + expected_result = False + result = daemon.runner.is_pidfile_stale(self.test_pidfile) + self.assertEqual(expected_result, result) + + def test_returns_false_if_process_exists(self): + """ Should return False if the process with its PID exists. """ + expected_result = False + result = daemon.runner.is_pidfile_stale(self.test_pidfile) + self.assertEqual(expected_result, result) + + def test_returns_true_if_process_does_not_exist(self): + """ Should return True if the process does not exist. """ + test_error = ProcessLookupError("No such process") + del os.kill.return_value + os.kill.side_effect = test_error + expected_result = True + result = daemon.runner.is_pidfile_stale(self.test_pidfile) + self.assertEqual(expected_result, result) + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/test_version.py b/scripts/external_libs/python-daemon-2.0.5/test_version.py new file mode 100644 index 00000000..b52f521d --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/test_version.py @@ -0,0 +1,1373 @@ +# -*- coding: utf-8 -*- +# +# test_version.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; version 3 of that license or any later version. +# No warranty expressed or implied. See the file ‘LICENSE.GPL-3’ for details. + +""" Unit test for ‘version’ packaging module. """ + +from __future__ import (absolute_import, unicode_literals) + +import os +import os.path +import io +import errno +import functools +import collections +import textwrap +import json +import tempfile +import distutils.dist +import distutils.cmd +import distutils.errors +import distutils.fancy_getopt +try: + # Standard library of Python 2.7 and later. + from io import StringIO +except ImportError: + # Standard library of Python 2.6 and earlier. + from StringIO import StringIO + +import mock +import testtools +import testscenarios +import docutils +import docutils.writers +import docutils.nodes +import setuptools +import setuptools.command + +import version + +version.ensure_class_bases_begin_with( + version.__dict__, str('VersionInfoWriter'), docutils.writers.Writer) +version.ensure_class_bases_begin_with( + version.__dict__, str('VersionInfoTranslator'), + docutils.nodes.SparseNodeVisitor) + + +def make_test_classes_for_ensure_class_bases_begin_with(): + """ Make test classes for use with ‘ensure_class_bases_begin_with’. + + :return: Mapping {`name`: `type`} of the custom types created. + + """ + + class quux_metaclass(type): + def __new__(metaclass, name, bases, namespace): + return super(quux_metaclass, metaclass).__new__( + metaclass, name, bases, namespace) + + class Foo(object): + __metaclass__ = type + + class Bar(object): + pass + + class FooInheritingBar(Bar): + __metaclass__ = type + + class FooWithCustomMetaclass(object): + __metaclass__ = quux_metaclass + + result = dict( + (name, value) for (name, value) in locals().items() + if isinstance(value, type)) + + return result + +class ensure_class_bases_begin_with_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘ensure_class_bases_begin_with’ function. """ + + test_classes = make_test_classes_for_ensure_class_bases_begin_with() + + scenarios = [ + ('simple', { + 'test_class': test_classes['Foo'], + 'base_class': test_classes['Bar'], + }), + ('custom metaclass', { + 'test_class': test_classes['FooWithCustomMetaclass'], + 'base_class': test_classes['Bar'], + 'expected_metaclass': test_classes['quux_metaclass'], + }), + ] + + def setUp(self): + """ Set up test fixtures. """ + super(ensure_class_bases_begin_with_TestCase, self).setUp() + + self.class_name = self.test_class.__name__ + self.test_module_namespace = {self.class_name: self.test_class} + + if not hasattr(self, 'expected_metaclass'): + self.expected_metaclass = type + + patcher_metaclass = mock.patch.object( + self.test_class, '__metaclass__') + patcher_metaclass.start() + self.addCleanup(patcher_metaclass.stop) + + self.fake_new_class = type(object) + self.test_class.__metaclass__.return_value = ( + self.fake_new_class) + + def test_module_namespace_contains_new_class(self): + """ Specified module namespace should have new class. """ + version.ensure_class_bases_begin_with( + self.test_module_namespace, self.class_name, self.base_class) + self.assertIn(self.fake_new_class, self.test_module_namespace.values()) + + def test_calls_metaclass_with_expected_class_name(self): + """ Should call the metaclass with the expected class name. """ + version.ensure_class_bases_begin_with( + self.test_module_namespace, self.class_name, self.base_class) + expected_class_name = self.class_name + self.test_class.__metaclass__.assert_called_with( + expected_class_name, mock.ANY, mock.ANY) + + def test_calls_metaclass_with_expected_bases(self): + """ Should call the metaclass with the expected bases. """ + version.ensure_class_bases_begin_with( + self.test_module_namespace, self.class_name, self.base_class) + expected_bases = tuple( + [self.base_class] + + list(self.test_class.__bases__)) + self.test_class.__metaclass__.assert_called_with( + mock.ANY, expected_bases, mock.ANY) + + def test_calls_metaclass_with_expected_namespace(self): + """ Should call the metaclass with the expected class namespace. """ + version.ensure_class_bases_begin_with( + self.test_module_namespace, self.class_name, self.base_class) + expected_namespace = self.test_class.__dict__.copy() + del expected_namespace['__dict__'] + self.test_class.__metaclass__.assert_called_with( + mock.ANY, mock.ANY, expected_namespace) + + +class ensure_class_bases_begin_with_AlreadyHasBase_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘ensure_class_bases_begin_with’ function. + + These test cases test the conditions where the class's base is + already the specified base class. + + """ + + test_classes = make_test_classes_for_ensure_class_bases_begin_with() + + scenarios = [ + ('already Bar subclass', { + 'test_class': test_classes['FooInheritingBar'], + 'base_class': test_classes['Bar'], + }), + ] + + def setUp(self): + """ Set up test fixtures. """ + super( + ensure_class_bases_begin_with_AlreadyHasBase_TestCase, + self).setUp() + + self.class_name = self.test_class.__name__ + self.test_module_namespace = {self.class_name: self.test_class} + + patcher_metaclass = mock.patch.object( + self.test_class, '__metaclass__') + patcher_metaclass.start() + self.addCleanup(patcher_metaclass.stop) + + def test_metaclass_not_called(self): + """ Should not call metaclass to create a new type. """ + version.ensure_class_bases_begin_with( + self.test_module_namespace, self.class_name, self.base_class) + self.assertFalse(self.test_class.__metaclass__.called) + + +class VersionInfoWriter_TestCase(testtools.TestCase): + """ Test cases for ‘VersionInfoWriter’ class. """ + + def setUp(self): + """ Set up test fixtures. """ + super(VersionInfoWriter_TestCase, self).setUp() + + self.test_instance = version.VersionInfoWriter() + + def test_declares_version_info_support(self): + """ Should declare support for ‘version_info’. """ + instance = self.test_instance + expected_support = "version_info" + result = instance.supports(expected_support) + self.assertTrue(result) + + +class VersionInfoWriter_translate_TestCase(testtools.TestCase): + """ Test cases for ‘VersionInfoWriter.translate’ method. """ + + def setUp(self): + """ Set up test fixtures. """ + super(VersionInfoWriter_translate_TestCase, self).setUp() + + patcher_translator = mock.patch.object( + version, 'VersionInfoTranslator') + self.mock_class_translator = patcher_translator.start() + self.addCleanup(patcher_translator.stop) + self.mock_translator = self.mock_class_translator.return_value + + self.test_instance = version.VersionInfoWriter() + patcher_document = mock.patch.object( + self.test_instance, 'document') + patcher_document.start() + self.addCleanup(patcher_document.stop) + + def test_creates_translator_with_document(self): + """ Should create a translator with the writer's document. """ + instance = self.test_instance + expected_document = self.test_instance.document + instance.translate() + self.mock_class_translator.assert_called_with(expected_document) + + def test_calls_document_walkabout_with_translator(self): + """ Should call document.walkabout with the translator. """ + instance = self.test_instance + instance.translate() + instance.document.walkabout.assert_called_with(self.mock_translator) + + def test_output_from_translator_astext(self): + """ Should have output from translator.astext(). """ + instance = self.test_instance + instance.translate() + expected_output = self.mock_translator.astext.return_value + self.assertEqual(expected_output, instance.output) + + +class ChangeLogEntry_TestCase(testtools.TestCase): + """ Test cases for ‘ChangeLogEntry’ class. """ + + def setUp(self): + """ Set up test fixtures. """ + super(ChangeLogEntry_TestCase, self).setUp() + + self.test_instance = version.ChangeLogEntry() + + def test_instantiate(self): + """ New instance of ‘ChangeLogEntry’ should be created. """ + self.assertIsInstance( + self.test_instance, version.ChangeLogEntry) + + def test_minimum_zero_arguments(self): + """ Initialiser should not require any arguments. """ + instance = version.ChangeLogEntry() + self.assertIsNot(instance, None) + + +class ChangeLogEntry_release_date_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘ChangeLogEntry.release_date’ attribute. """ + + scenarios = [ + ('default', { + 'test_args': {}, + 'expected_release_date': + version.ChangeLogEntry.default_release_date, + }), + ('unknown token', { + 'test_args': {'release_date': "UNKNOWN"}, + 'expected_release_date': "UNKNOWN", + }), + ('future token', { + 'test_args': {'release_date': "FUTURE"}, + 'expected_release_date': "FUTURE", + }), + ('2001-01-01', { + 'test_args': {'release_date': "2001-01-01"}, + 'expected_release_date': "2001-01-01", + }), + ('bogus', { + 'test_args': {'release_date': "b0gUs"}, + 'expected_error': ValueError, + }), + ] + + def test_has_expected_release_date(self): + """ Should have default `release_date` attribute. """ + if hasattr(self, 'expected_error'): + self.assertRaises( + self.expected_error, + version.ChangeLogEntry, **self.test_args) + else: + instance = version.ChangeLogEntry(**self.test_args) + self.assertEqual(self.expected_release_date, instance.release_date) + + +class ChangeLogEntry_version_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘ChangeLogEntry.version’ attribute. """ + + scenarios = [ + ('default', { + 'test_args': {}, + 'expected_version': + version.ChangeLogEntry.default_version, + }), + ('unknown token', { + 'test_args': {'version': "UNKNOWN"}, + 'expected_version': "UNKNOWN", + }), + ('0.0', { + 'test_args': {'version': "0.0"}, + 'expected_version': "0.0", + }), + ] + + def test_has_expected_version(self): + """ Should have default `version` attribute. """ + instance = version.ChangeLogEntry(**self.test_args) + self.assertEqual(self.expected_version, instance.version) + + +class ChangeLogEntry_maintainer_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘ChangeLogEntry.maintainer’ attribute. """ + + scenarios = [ + ('default', { + 'test_args': {}, + 'expected_maintainer': None, + }), + ('person', { + 'test_args': {'maintainer': "Foo Bar <foo.bar@example.org>"}, + 'expected_maintainer': "Foo Bar <foo.bar@example.org>", + }), + ('bogus', { + 'test_args': {'maintainer': "b0gUs"}, + 'expected_error': ValueError, + }), + ] + + def test_has_expected_maintainer(self): + """ Should have default `maintainer` attribute. """ + if hasattr(self, 'expected_error'): + self.assertRaises( + self.expected_error, + version.ChangeLogEntry, **self.test_args) + else: + instance = version.ChangeLogEntry(**self.test_args) + self.assertEqual(self.expected_maintainer, instance.maintainer) + + +class ChangeLogEntry_body_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘ChangeLogEntry.body’ attribute. """ + + scenarios = [ + ('default', { + 'test_args': {}, + 'expected_body': None, + }), + ('simple', { + 'test_args': {'body': "Foo bar baz."}, + 'expected_body': "Foo bar baz.", + }), + ] + + def test_has_expected_body(self): + """ Should have default `body` attribute. """ + instance = version.ChangeLogEntry(**self.test_args) + self.assertEqual(self.expected_body, instance.body) + + +class ChangeLogEntry_as_version_info_entry_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘ChangeLogEntry.as_version_info_entry’ attribute. """ + + scenarios = [ + ('default', { + 'test_args': {}, + 'expected_result': collections.OrderedDict([ + ('release_date', version.ChangeLogEntry.default_release_date), + ('version', version.ChangeLogEntry.default_version), + ('maintainer', None), + ('body', None), + ]), + }), + ] + + def setUp(self): + """ Set up test fixtures. """ + super(ChangeLogEntry_as_version_info_entry_TestCase, self).setUp() + + self.test_instance = version.ChangeLogEntry(**self.test_args) + + def test_returns_result(self): + """ Should return expected result. """ + result = self.test_instance.as_version_info_entry() + self.assertEqual(self.expected_result, result) + + +def make_mock_field_node(field_name, field_body): + """ Make a mock Docutils field node for tests. """ + + mock_field_node = mock.MagicMock( + name='field', spec=docutils.nodes.field) + + mock_field_name_node = mock.MagicMock( + name='field_name', spec=docutils.nodes.field_name) + mock_field_name_node.parent = mock_field_node + mock_field_name_node.children = [field_name] + + mock_field_body_node = mock.MagicMock( + name='field_body', spec=docutils.nodes.field_body) + mock_field_body_node.parent = mock_field_node + mock_field_body_node.children = [field_body] + + mock_field_node.children = [mock_field_name_node, mock_field_body_node] + + def fake_func_first_child_matching_class(node_class): + result = None + node_class_name = node_class.__name__ + for (index, node) in enumerate(mock_field_node.children): + if node._mock_name == node_class_name: + result = index + break + return result + + mock_field_node.first_child_matching_class.side_effect = ( + fake_func_first_child_matching_class) + + return mock_field_node + + +class JsonEqual(testtools.matchers.Matcher): + """ A matcher to compare the value of JSON streams. """ + + def __init__(self, expected): + self.expected_value = expected + + def match(self, content): + """ Assert the JSON `content` matches the `expected_content`. """ + result = None + actual_value = json.loads(content.decode('utf-8')) + if actual_value != self.expected_value: + result = JsonValueMismatch(self.expected_value, actual_value) + return result + + +class JsonValueMismatch(testtools.matchers.Mismatch): + """ The specified JSON stream does not evaluate to the expected value. """ + + def __init__(self, expected, actual): + self.expected_value = expected + self.actual_value = actual + + def describe(self): + """ Emit a text description of this mismatch. """ + expected_json_text = json.dumps(self.expected_value, indent=4) + actual_json_text = json.dumps(self.actual_value, indent=4) + text = ( + "\n" + "reference: {expected}\n" + "actual: {actual}\n").format( + expected=expected_json_text, actual=actual_json_text) + return text + + +class changelog_to_version_info_collection_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘changelog_to_version_info_collection’ function. """ + + scenarios = [ + ('single entry', { + 'test_input': textwrap.dedent("""\ + Version 1.0 + =========== + + :Released: 2009-01-01 + :Maintainer: Foo Bar <foo.bar@example.org> + + * Lorem ipsum dolor sit amet. + """), + 'expected_version_info': [ + { + 'release_date': "2009-01-01", + 'version': "1.0", + 'maintainer': "Foo Bar <foo.bar@example.org>", + 'body': "* Lorem ipsum dolor sit amet.\n", + }, + ], + }), + ('multiple entries', { + 'test_input': textwrap.dedent("""\ + Version 1.0 + =========== + + :Released: 2009-01-01 + :Maintainer: Foo Bar <foo.bar@example.org> + + * Lorem ipsum dolor sit amet. + + + Version 0.8 + =========== + + :Released: 2004-01-01 + :Maintainer: Foo Bar <foo.bar@example.org> + + * Donec venenatis nisl aliquam ipsum. + + + Version 0.7.2 + ============= + + :Released: 2001-01-01 + :Maintainer: Foo Bar <foo.bar@example.org> + + * Pellentesque elementum mollis finibus. + """), + 'expected_version_info': [ + { + 'release_date': "2009-01-01", + 'version': "1.0", + 'maintainer': "Foo Bar <foo.bar@example.org>", + 'body': "* Lorem ipsum dolor sit amet.\n", + }, + { + 'release_date': "2004-01-01", + 'version': "0.8", + 'maintainer': "Foo Bar <foo.bar@example.org>", + 'body': "* Donec venenatis nisl aliquam ipsum.\n", + }, + { + 'release_date': "2001-01-01", + 'version': "0.7.2", + 'maintainer': "Foo Bar <foo.bar@example.org>", + 'body': "* Pellentesque elementum mollis finibus.\n", + }, + ], + }), + ('trailing comment', { + 'test_input': textwrap.dedent("""\ + Version NEXT + ============ + + :Released: FUTURE + :Maintainer: + + * Lorem ipsum dolor sit amet. + + .. + Vivamus aliquam felis rutrum rutrum dictum. + """), + 'expected_version_info': [ + { + 'release_date': "FUTURE", + 'version': "NEXT", + 'maintainer': "", + 'body': "* Lorem ipsum dolor sit amet.\n", + }, + ], + }), + ('inline comment', { + 'test_input': textwrap.dedent("""\ + Version NEXT + ============ + + :Released: FUTURE + :Maintainer: + + .. + Vivamus aliquam felis rutrum rutrum dictum. + + * Lorem ipsum dolor sit amet. + """), + 'expected_version_info': [ + { + 'release_date': "FUTURE", + 'version': "NEXT", + 'maintainer': "", + 'body': "* Lorem ipsum dolor sit amet.\n", + }, + ], + }), + ('unreleased entry', { + 'test_input': textwrap.dedent("""\ + Version NEXT + ============ + + :Released: FUTURE + :Maintainer: + + * Lorem ipsum dolor sit amet. + + + Version 0.8 + =========== + + :Released: 2001-01-01 + :Maintainer: Foo Bar <foo.bar@example.org> + + * Donec venenatis nisl aliquam ipsum. + """), + 'expected_version_info': [ + { + 'release_date': "FUTURE", + 'version': "NEXT", + 'maintainer': "", + 'body': "* Lorem ipsum dolor sit amet.\n", + }, + { + 'release_date': "2001-01-01", + 'version': "0.8", + 'maintainer': "Foo Bar <foo.bar@example.org>", + 'body': "* Donec venenatis nisl aliquam ipsum.\n", + }, + ], + }), + ('no section', { + 'test_input': textwrap.dedent("""\ + :Released: 2009-01-01 + :Maintainer: Foo Bar <foo.bar@example.org> + + * Lorem ipsum dolor sit amet. + """), + 'expected_error': version.InvalidFormatError, + }), + ('subsection', { + 'test_input': textwrap.dedent("""\ + Version 1.0 + =========== + + :Released: 2009-01-01 + :Maintainer: Foo Bar <foo.bar@example.org> + + * Lorem ipsum dolor sit amet. + + Ut ultricies fermentum quam + --------------------------- + + * In commodo magna facilisis in. + """), + 'expected_error': version.InvalidFormatError, + 'subsection': True, + }), + ('unknown field', { + 'test_input': textwrap.dedent("""\ + Version 1.0 + =========== + + :Released: 2009-01-01 + :Maintainer: Foo Bar <foo.bar@example.org> + :Favourite: Spam + + * Lorem ipsum dolor sit amet. + """), + 'expected_error': version.InvalidFormatError, + }), + ('invalid version word', { + 'test_input': textwrap.dedent("""\ + BoGuS 1.0 + ========= + + :Released: 2009-01-01 + :Maintainer: Foo Bar <foo.bar@example.org> + + * Lorem ipsum dolor sit amet. + """), + 'expected_error': version.InvalidFormatError, + }), + ('invalid section title', { + 'test_input': textwrap.dedent("""\ + Lorem Ipsum 1.0 + =============== + + :Released: 2009-01-01 + :Maintainer: Foo Bar <foo.bar@example.org> + + * Lorem ipsum dolor sit amet. + """), + 'expected_error': version.InvalidFormatError, + }), + ] + + def test_returns_expected_version_info(self): + """ Should return expected version info mapping. """ + infile = StringIO(self.test_input) + if hasattr(self, 'expected_error'): + self.assertRaises( + self.expected_error, + version.changelog_to_version_info_collection, infile) + else: + result = version.changelog_to_version_info_collection(infile) + self.assertThat(result, JsonEqual(self.expected_version_info)) + + +try: + FileNotFoundError + PermissionError +except NameError: + # Python 2 uses OSError. + FileNotFoundError = functools.partial(IOError, errno.ENOENT) + PermissionError = functools.partial(IOError, errno.EPERM) + +fake_version_info = { + 'release_date': "2001-01-01", 'version': "2.0", + 'maintainer': None, 'body': None, + } + +@mock.patch.object( + version, "get_latest_version", return_value=fake_version_info) +class generate_version_info_from_changelog_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘generate_version_info_from_changelog’ function. """ + + fake_open_side_effects = { + 'success': ( + lambda *args, **kwargs: StringIO()), + 'file not found': FileNotFoundError(), + 'permission denied': PermissionError(), + } + + scenarios = [ + ('simple', { + 'open_scenario': 'success', + 'fake_versions_json': json.dumps([fake_version_info]), + 'expected_result': fake_version_info, + }), + ('file not found', { + 'open_scenario': 'file not found', + 'expected_result': {}, + }), + ('permission denied', { + 'open_scenario': 'permission denied', + 'expected_result': {}, + }), + ] + + def setUp(self): + """ Set up test fixtures. """ + super(generate_version_info_from_changelog_TestCase, self).setUp() + + self.fake_changelog_file_path = tempfile.mktemp() + + def fake_open(filespec, *args, **kwargs): + if filespec == self.fake_changelog_file_path: + side_effect = self.fake_open_side_effects[self.open_scenario] + if callable(side_effect): + result = side_effect() + else: + raise side_effect + else: + result = StringIO() + return result + + func_patcher_io_open = mock.patch.object( + io, "open") + func_patcher_io_open.start() + self.addCleanup(func_patcher_io_open.stop) + io.open.side_effect = fake_open + + self.file_encoding = "utf-8" + + func_patcher_changelog_to_version_info_collection = mock.patch.object( + version, "changelog_to_version_info_collection") + func_patcher_changelog_to_version_info_collection.start() + self.addCleanup(func_patcher_changelog_to_version_info_collection.stop) + if hasattr(self, 'fake_versions_json'): + version.changelog_to_version_info_collection.return_value = ( + self.fake_versions_json.encode(self.file_encoding)) + + def test_returns_empty_collection_on_read_error( + self, + mock_func_get_latest_version): + """ Should return empty collection on error reading changelog. """ + test_error = PermissionError("Not for you") + version.changelog_to_version_info_collection.side_effect = test_error + result = version.generate_version_info_from_changelog( + self.fake_changelog_file_path) + expected_result = {} + self.assertDictEqual(expected_result, result) + + def test_opens_file_with_expected_encoding( + self, + mock_func_get_latest_version): + """ Should open changelog file in text mode with expected encoding. """ + result = version.generate_version_info_from_changelog( + self.fake_changelog_file_path) + expected_file_path = self.fake_changelog_file_path + expected_open_mode = 'rt' + expected_encoding = self.file_encoding + (open_args_positional, open_args_kwargs) = io.open.call_args + (open_args_filespec, open_args_mode) = open_args_positional[:2] + open_args_encoding = open_args_kwargs['encoding'] + self.assertEqual(expected_file_path, open_args_filespec) + self.assertEqual(expected_open_mode, open_args_mode) + self.assertEqual(expected_encoding, open_args_encoding) + + def test_returns_expected_result( + self, + mock_func_get_latest_version): + """ Should return expected result. """ + result = version.generate_version_info_from_changelog( + self.fake_changelog_file_path) + self.assertEqual(self.expected_result, result) + + +DefaultNoneDict = functools.partial(collections.defaultdict, lambda: None) + +class get_latest_version_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘get_latest_version’ function. """ + + scenarios = [ + ('simple', { + 'test_versions': [ + DefaultNoneDict({'release_date': "LATEST"}), + ], + 'expected_result': version.ChangeLogEntry.make_ordered_dict( + DefaultNoneDict({'release_date': "LATEST"})), + }), + ('no versions', { + 'test_versions': [], + 'expected_result': collections.OrderedDict(), + }), + ('ordered versions', { + 'test_versions': [ + DefaultNoneDict({'release_date': "1"}), + DefaultNoneDict({'release_date': "2"}), + DefaultNoneDict({'release_date': "LATEST"}), + ], + 'expected_result': version.ChangeLogEntry.make_ordered_dict( + DefaultNoneDict({'release_date': "LATEST"})), + }), + ('un-ordered versions', { + 'test_versions': [ + DefaultNoneDict({'release_date': "2"}), + DefaultNoneDict({'release_date': "LATEST"}), + DefaultNoneDict({'release_date': "1"}), + ], + 'expected_result': version.ChangeLogEntry.make_ordered_dict( + DefaultNoneDict({'release_date': "LATEST"})), + }), + ] + + def test_returns_expected_result(self): + """ Should return expected result. """ + result = version.get_latest_version(self.test_versions) + self.assertDictEqual(self.expected_result, result) + + +@mock.patch.object(json, "dumps", side_effect=json.dumps) +class serialise_version_info_from_mapping_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘get_latest_version’ function. """ + + scenarios = [ + ('simple', { + 'test_version_info': {'foo': "spam"}, + }), + ] + + for (name, scenario) in scenarios: + scenario['fake_json_dump'] = json.dumps(scenario['test_version_info']) + scenario['expected_value'] = scenario['test_version_info'] + + def test_passes_specified_object(self, mock_func_json_dumps): + """ Should pass the specified object to `json.dumps`. """ + result = version.serialise_version_info_from_mapping( + self.test_version_info) + mock_func_json_dumps.assert_called_with( + self.test_version_info, indent=mock.ANY) + + def test_returns_expected_result(self, mock_func_json_dumps): + """ Should return expected result. """ + mock_func_json_dumps.return_value = self.fake_json_dump + result = version.serialise_version_info_from_mapping( + self.test_version_info) + value = json.loads(result) + self.assertEqual(self.expected_value, value) + + +DistributionMetadata_defaults = { + name: None + for name in list(collections.OrderedDict.fromkeys( + distutils.dist.DistributionMetadata._METHOD_BASENAMES))} +FakeDistributionMetadata = collections.namedtuple( + 'FakeDistributionMetadata', DistributionMetadata_defaults.keys()) + +Distribution_defaults = { + 'metadata': None, + 'version': None, + 'release_date': None, + 'maintainer': None, + 'maintainer_email': None, + } +FakeDistribution = collections.namedtuple( + 'FakeDistribution', Distribution_defaults.keys()) + +def make_fake_distribution( + fields_override=None, metadata_fields_override=None): + metadata_fields = DistributionMetadata_defaults.copy() + if metadata_fields_override is not None: + metadata_fields.update(metadata_fields_override) + metadata = FakeDistributionMetadata(**metadata_fields) + + fields = Distribution_defaults.copy() + fields['metadata'] = metadata + if fields_override is not None: + fields.update(fields_override) + distribution = FakeDistribution(**fields) + + return distribution + + +class get_changelog_path_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘get_changelog_path’ function. """ + + default_path = "." + default_script_filename = "setup.py" + + scenarios = [ + ('simple', {}), + ('unusual script name', { + 'script_filename': "lorem_ipsum", + }), + ('relative script path', { + 'script_directory': "dolor/sit/amet", + }), + ('absolute script path', { + 'script_directory': "/dolor/sit/amet", + }), + ('specify filename', { + 'changelog_filename': "adipiscing", + }), + ] + + def setUp(self): + """ Set up test fixtures. """ + super(get_changelog_path_TestCase, self).setUp() + + self.test_distribution = mock.MagicMock(distutils.dist.Distribution) + + if not hasattr(self, 'script_directory'): + self.script_directory = self.default_path + if not hasattr(self, 'script_filename'): + self.script_filename = self.default_script_filename + self.test_distribution.script_name = os.path.join( + self.script_directory, self.script_filename) + + changelog_filename = version.changelog_filename + if hasattr(self, 'changelog_filename'): + changelog_filename = self.changelog_filename + + self.expected_result = os.path.join( + self.script_directory, changelog_filename) + + def test_returns_expected_result(self): + """ Should return expected result. """ + args = { + 'distribution': self.test_distribution, + } + if hasattr(self, 'changelog_filename'): + args.update({'filename': self.changelog_filename}) + result = version.get_changelog_path(**args) + self.assertEqual(self.expected_result, result) + + +class WriteVersionInfoCommand_BaseTestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Base class for ‘WriteVersionInfoCommand’ test case classes. """ + + def setUp(self): + """ Set up test fixtures. """ + super(WriteVersionInfoCommand_BaseTestCase, self).setUp() + + fake_distribution_name = self.getUniqueString() + + self.test_distribution = distutils.dist.Distribution() + self.test_distribution.metadata.name = fake_distribution_name + + +class WriteVersionInfoCommand_TestCase(WriteVersionInfoCommand_BaseTestCase): + """ Test cases for ‘WriteVersionInfoCommand’ class. """ + + def test_subclass_of_distutils_command(self): + """ Should be a subclass of ‘distutils.cmd.Command’. """ + instance = version.WriteVersionInfoCommand(self.test_distribution) + self.assertIsInstance(instance, distutils.cmd.Command) + + +class WriteVersionInfoCommand_user_options_TestCase( + WriteVersionInfoCommand_BaseTestCase): + """ Test cases for ‘WriteVersionInfoCommand.user_options’ attribute. """ + + def setUp(self): + """ Set up test fixtures. """ + super(WriteVersionInfoCommand_user_options_TestCase, self).setUp() + + self.test_instance = version.WriteVersionInfoCommand( + self.test_distribution) + self.commandline_parser = distutils.fancy_getopt.FancyGetopt( + self.test_instance.user_options) + + def test_parses_correctly_as_fancy_getopt(self): + """ Should parse correctly in ‘FancyGetopt’. """ + self.assertIsInstance( + self.commandline_parser, distutils.fancy_getopt.FancyGetopt) + + def test_includes_base_class_user_options(self): + """ Should include base class's user_options. """ + base_command = setuptools.command.egg_info.egg_info + expected_user_options = base_command.user_options + self.assertThat( + set(expected_user_options), + IsSubset(set(self.test_instance.user_options))) + + def test_has_option_changelog_path(self): + """ Should have a ‘changelog-path’ option. """ + expected_option_name = "changelog-path=" + result = self.commandline_parser.has_option(expected_option_name) + self.assertTrue(result) + + def test_has_option_outfile_path(self): + """ Should have a ‘outfile-path’ option. """ + expected_option_name = "outfile-path=" + result = self.commandline_parser.has_option(expected_option_name) + self.assertTrue(result) + + +class WriteVersionInfoCommand_initialize_options_TestCase( + WriteVersionInfoCommand_BaseTestCase): + """ Test cases for ‘WriteVersionInfoCommand.initialize_options’ method. """ + + def setUp(self): + """ Set up test fixtures. """ + super( + WriteVersionInfoCommand_initialize_options_TestCase, self + ).setUp() + + patcher_func_egg_info_initialize_options = mock.patch.object( + setuptools.command.egg_info.egg_info, "initialize_options") + patcher_func_egg_info_initialize_options.start() + self.addCleanup(patcher_func_egg_info_initialize_options.stop) + + def test_calls_base_class_method(self): + """ Should call base class's ‘initialize_options’ method. """ + instance = version.WriteVersionInfoCommand(self.test_distribution) + base_command_class = setuptools.command.egg_info.egg_info + base_command_class.initialize_options.assert_called_with() + + def test_sets_changelog_path_to_none(self): + """ Should set ‘changelog_path’ attribute to ``None``. """ + instance = version.WriteVersionInfoCommand(self.test_distribution) + self.assertIs(instance.changelog_path, None) + + def test_sets_outfile_path_to_none(self): + """ Should set ‘outfile_path’ attribute to ``None``. """ + instance = version.WriteVersionInfoCommand(self.test_distribution) + self.assertIs(instance.outfile_path, None) + + +class WriteVersionInfoCommand_finalize_options_TestCase( + WriteVersionInfoCommand_BaseTestCase): + """ Test cases for ‘WriteVersionInfoCommand.finalize_options’ method. """ + + def setUp(self): + """ Set up test fixtures. """ + super(WriteVersionInfoCommand_finalize_options_TestCase, self).setUp() + + self.test_instance = version.WriteVersionInfoCommand(self.test_distribution) + + patcher_func_egg_info_finalize_options = mock.patch.object( + setuptools.command.egg_info.egg_info, "finalize_options") + patcher_func_egg_info_finalize_options.start() + self.addCleanup(patcher_func_egg_info_finalize_options.stop) + + self.fake_script_dir = self.getUniqueString() + self.test_distribution.script_name = os.path.join( + self.fake_script_dir, self.getUniqueString()) + + self.fake_egg_dir = self.getUniqueString() + self.test_instance.egg_info = self.fake_egg_dir + + patcher_func_get_changelog_path = mock.patch.object( + version, "get_changelog_path") + patcher_func_get_changelog_path.start() + self.addCleanup(patcher_func_get_changelog_path.stop) + + self.fake_changelog_path = self.getUniqueString() + version.get_changelog_path.return_value = self.fake_changelog_path + + def test_calls_base_class_method(self): + """ Should call base class's ‘finalize_options’ method. """ + base_command_class = setuptools.command.egg_info.egg_info + self.test_instance.finalize_options() + base_command_class.finalize_options.assert_called_with() + + def test_sets_force_to_none(self): + """ Should set ‘force’ attribute to ``None``. """ + self.test_instance.finalize_options() + self.assertIs(self.test_instance.force, None) + + def test_sets_changelog_path_using_get_changelog_path(self): + """ Should set ‘changelog_path’ attribute if it was ``None``. """ + self.test_instance.changelog_path = None + self.test_instance.finalize_options() + expected_changelog_path = self.fake_changelog_path + self.assertEqual(expected_changelog_path, self.test_instance.changelog_path) + + def test_leaves_changelog_path_if_already_set(self): + """ Should leave ‘changelog_path’ attribute set. """ + prior_changelog_path = self.getUniqueString() + self.test_instance.changelog_path = prior_changelog_path + self.test_instance.finalize_options() + expected_changelog_path = prior_changelog_path + self.assertEqual(expected_changelog_path, self.test_instance.changelog_path) + + def test_sets_outfile_path_to_default(self): + """ Should set ‘outfile_path’ attribute to default value. """ + fake_version_info_filename = self.getUniqueString() + with mock.patch.object( + version, "version_info_filename", + new=fake_version_info_filename): + self.test_instance.finalize_options() + expected_outfile_path = os.path.join( + self.fake_egg_dir, fake_version_info_filename) + self.assertEqual(expected_outfile_path, self.test_instance.outfile_path) + + def test_leaves_outfile_path_if_already_set(self): + """ Should leave ‘outfile_path’ attribute set. """ + prior_outfile_path = self.getUniqueString() + self.test_instance.outfile_path = prior_outfile_path + self.test_instance.finalize_options() + expected_outfile_path = prior_outfile_path + self.assertEqual(expected_outfile_path, self.test_instance.outfile_path) + + +class has_changelog_TestCase( + testscenarios.WithScenarios, testtools.TestCase): + """ Test cases for ‘has_changelog’ function. """ + + fake_os_path_exists_side_effects = { + 'true': (lambda path: True), + 'false': (lambda path: False), + } + + scenarios = [ + ('no changelog path', { + 'changelog_path': None, + 'expected_result': False, + }), + ('changelog exists', { + 'os_path_exists_scenario': 'true', + 'expected_result': True, + }), + ('changelog not found', { + 'os_path_exists_scenario': 'false', + 'expected_result': False, + }), + ] + + def setUp(self): + """ Set up test fixtures. """ + super(has_changelog_TestCase, self).setUp() + + self.test_distribution = distutils.dist.Distribution() + self.test_command = version.EggInfoCommand( + self.test_distribution) + + patcher_func_get_changelog_path = mock.patch.object( + version, "get_changelog_path") + patcher_func_get_changelog_path.start() + self.addCleanup(patcher_func_get_changelog_path.stop) + + self.fake_changelog_file_path = self.getUniqueString() + if hasattr(self, 'changelog_path'): + self.fake_changelog_file_path = self.changelog_path + version.get_changelog_path.return_value = self.fake_changelog_file_path + self.fake_changelog_file = StringIO() + + def fake_os_path_exists(path): + if path == self.fake_changelog_file_path: + side_effect = self.fake_os_path_exists_side_effects[ + self.os_path_exists_scenario] + if callable(side_effect): + result = side_effect(path) + else: + raise side_effect + else: + result = False + return result + + func_patcher_os_path_exists = mock.patch.object( + os.path, "exists") + func_patcher_os_path_exists.start() + self.addCleanup(func_patcher_os_path_exists.stop) + os.path.exists.side_effect = fake_os_path_exists + + def test_gets_changelog_path_from_distribution(self): + """ Should call ‘get_changelog_path’ with distribution. """ + result = version.has_changelog(self.test_command) + version.get_changelog_path.assert_called_with( + self.test_distribution) + + def test_returns_expected_result(self): + """ Should be a subclass of ‘distutils.cmd.Command’. """ + result = version.has_changelog(self.test_command) + self.assertEqual(self.expected_result, result) + + +@mock.patch.object(version, 'generate_version_info_from_changelog') +@mock.patch.object(version, 'serialise_version_info_from_mapping') +@mock.patch.object(version.EggInfoCommand, "write_file") +class WriteVersionInfoCommand_run_TestCase( + WriteVersionInfoCommand_BaseTestCase): + """ Test cases for ‘WriteVersionInfoCommand.run’ method. """ + + def setUp(self): + """ Set up test fixtures. """ + super(WriteVersionInfoCommand_run_TestCase, self).setUp() + + self.test_instance = version.WriteVersionInfoCommand( + self.test_distribution) + + self.fake_changelog_path = self.getUniqueString() + self.test_instance.changelog_path = self.fake_changelog_path + + self.fake_outfile_path = self.getUniqueString() + self.test_instance.outfile_path = self.fake_outfile_path + + def test_returns_none( + self, + mock_func_egg_info_write_file, + mock_func_serialise_version_info, + mock_func_generate_version_info): + """ Should return ``None``. """ + result = self.test_instance.run() + self.assertIs(result, None) + + def test_generates_version_info_from_changelog( + self, + mock_func_egg_info_write_file, + mock_func_serialise_version_info, + mock_func_generate_version_info): + """ Should generate version info from specified changelog. """ + self.test_instance.run() + expected_changelog_path = self.test_instance.changelog_path + mock_func_generate_version_info.assert_called_with( + expected_changelog_path) + + def test_serialises_version_info_from_mapping( + self, + mock_func_egg_info_write_file, + mock_func_serialise_version_info, + mock_func_generate_version_info): + """ Should serialise version info from specified mapping. """ + self.test_instance.run() + expected_version_info = mock_func_generate_version_info.return_value + mock_func_serialise_version_info.assert_called_with( + expected_version_info) + + def test_writes_file_using_command_context( + self, + mock_func_egg_info_write_file, + mock_func_serialise_version_info, + mock_func_generate_version_info): + """ Should write the metadata file using the command context. """ + self.test_instance.run() + expected_content = mock_func_serialise_version_info.return_value + mock_func_egg_info_write_file.assert_called_with( + "version info", self.fake_outfile_path, expected_content) + + +IsSubset = testtools.matchers.MatchesPredicateWithParams( + set.issubset, "{0} should be a subset of {1}") + +class EggInfoCommand_TestCase(testtools.TestCase): + """ Test cases for ‘EggInfoCommand’ class. """ + + def setUp(self): + """ Set up test fixtures. """ + super(EggInfoCommand_TestCase, self).setUp() + + self.test_distribution = distutils.dist.Distribution() + self.test_instance = version.EggInfoCommand(self.test_distribution) + + def test_subclass_of_setuptools_egg_info(self): + """ Should be a subclass of Setuptools ‘egg_info’. """ + self.assertIsInstance( + self.test_instance, setuptools.command.egg_info.egg_info) + + def test_sub_commands_include_base_class_sub_commands(self): + """ Should include base class's sub-commands in this sub_commands. """ + base_command = setuptools.command.egg_info.egg_info + expected_sub_commands = base_command.sub_commands + self.assertThat( + set(expected_sub_commands), + IsSubset(set(self.test_instance.sub_commands))) + + def test_sub_commands_includes_write_version_info_command(self): + """ Should include sub-command named ‘write_version_info’. """ + commands_by_name = dict(self.test_instance.sub_commands) + expected_predicate = version.has_changelog + expected_item = ('write_version_info', expected_predicate) + self.assertIn(expected_item, commands_by_name.items()) + + +@mock.patch.object(setuptools.command.egg_info.egg_info, "run") +class EggInfoCommand_run_TestCase(testtools.TestCase): + """ Test cases for ‘EggInfoCommand.run’ method. """ + + def setUp(self): + """ Set up test fixtures. """ + super(EggInfoCommand_run_TestCase, self).setUp() + + self.test_distribution = distutils.dist.Distribution() + self.test_instance = version.EggInfoCommand(self.test_distribution) + + base_command = setuptools.command.egg_info.egg_info + patcher_func_egg_info_get_sub_commands = mock.patch.object( + base_command, "get_sub_commands") + patcher_func_egg_info_get_sub_commands.start() + self.addCleanup(patcher_func_egg_info_get_sub_commands.stop) + + patcher_func_egg_info_run_command = mock.patch.object( + base_command, "run_command") + patcher_func_egg_info_run_command.start() + self.addCleanup(patcher_func_egg_info_run_command.stop) + + self.fake_sub_commands = ["spam", "eggs", "beans"] + base_command.get_sub_commands.return_value = self.fake_sub_commands + + def test_returns_none(self, mock_func_egg_info_run): + """ Should return ``None``. """ + result = self.test_instance.run() + self.assertIs(result, None) + + def test_runs_each_command_in_sub_commands( + self, mock_func_egg_info_run): + """ Should run each command in ‘self.get_sub_commands()’. """ + base_command = setuptools.command.egg_info.egg_info + self.test_instance.run() + expected_calls = [mock.call(name) for name in self.fake_sub_commands] + base_command.run_command.assert_has_calls(expected_calls) + + def test_calls_base_class_run(self, mock_func_egg_info_run): + """ Should call base class's ‘run’ method. """ + result = self.test_instance.run() + mock_func_egg_info_run.assert_called_with() + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/python-daemon-2.0.5/version.py b/scripts/external_libs/python-daemon-2.0.5/version.py new file mode 100644 index 00000000..7e4c4202 --- /dev/null +++ b/scripts/external_libs/python-daemon-2.0.5/version.py @@ -0,0 +1,547 @@ +# -*- coding: utf-8 -*- + +# version.py +# Part of ‘python-daemon’, an implementation of PEP 3143. +# +# Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au> +# +# This is free software: you may copy, modify, and/or distribute this work +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; version 3 of that license or any later version. +# No warranty expressed or implied. See the file ‘LICENSE.GPL-3’ for details. + +""" Version information unified for human- and machine-readable formats. + + The project ‘ChangeLog’ file is a reStructuredText document, with + each section describing a version of the project. The document is + intended to be readable as-is by end users. + + This module handles transformation from the ‘ChangeLog’ to a + mapping of version information, serialised as JSON. It also + provides functionality for Distutils to use this information. + + Requires: + + * Docutils <http://docutils.sourceforge.net/> + * JSON <https://docs.python.org/3/reference/json.html> + + """ + +from __future__ import (absolute_import, unicode_literals) + +import sys +import os +import io +import errno +import json +import datetime +import textwrap +import re +import functools +import collections +import distutils +import distutils.errors +import distutils.cmd +try: + # Python 2 has both ‘str’ (bytes) and ‘unicode’ (text). + basestring = basestring + unicode = unicode +except NameError: + # Python 3 names the Unicode data type ‘str’. + basestring = str + unicode = str + +import setuptools +import setuptools.command.egg_info + + +def ensure_class_bases_begin_with(namespace, class_name, base_class): + """ Ensure the named class's bases start with the base class. + + :param namespace: The namespace containing the class name. + :param class_name: The name of the class to alter. + :param base_class: The type to be the first base class for the + newly created type. + :return: ``None``. + + This function is a hack to circumvent a circular dependency: + using classes from a module which is not installed at the time + this module is imported. + + Call this function after ensuring `base_class` is available, + before using the class named by `class_name`. + + """ + existing_class = namespace[class_name] + assert isinstance(existing_class, type) + + bases = list(existing_class.__bases__) + if base_class is bases[0]: + # Already bound to a type with the right bases. + return + bases.insert(0, base_class) + + new_class_namespace = existing_class.__dict__.copy() + # Type creation will assign the correct ‘__dict__’ attribute. + del new_class_namespace['__dict__'] + + metaclass = existing_class.__metaclass__ + new_class = metaclass(class_name, tuple(bases), new_class_namespace) + + namespace[class_name] = new_class + + +class VersionInfoWriter(object): + """ Docutils writer to produce a version info JSON data stream. """ + + # This class needs its base class to be a class from `docutils`. + # But that would create a circular dependency: Setuptools cannot + # ensure `docutils` is available before importing this module. + # + # Use `ensure_class_bases_begin_with` after importing `docutils`, to + # re-bind the `VersionInfoWriter` name to a new type that inherits + # from `docutils.writers.Writer`. + + __metaclass__ = type + + supported = ['version_info'] + """ Formats this writer supports. """ + + def __init__(self): + super(VersionInfoWriter, self).__init__() + self.translator_class = VersionInfoTranslator + + def translate(self): + visitor = self.translator_class(self.document) + self.document.walkabout(visitor) + self.output = visitor.astext() + + +rfc822_person_regex = re.compile( + "^(?P<name>[^<]+) <(?P<email>[^>]+)>$") + +class ChangeLogEntry: + """ An individual entry from the ‘ChangeLog’ document. """ + + __metaclass__ = type + + field_names = [ + 'release_date', + 'version', + 'maintainer', + 'body', + ] + + date_format = "%Y-%m-%d" + default_version = "UNKNOWN" + default_release_date = "UNKNOWN" + + def __init__( + self, + release_date=default_release_date, version=default_version, + maintainer=None, body=None): + self.validate_release_date(release_date) + self.release_date = release_date + + self.version = version + + self.validate_maintainer(maintainer) + self.maintainer = maintainer + self.body = body + + @classmethod + def validate_release_date(cls, value): + """ Validate the `release_date` value. + + :param value: The prospective `release_date` value. + :return: ``None`` if the value is valid. + :raises ValueError: If the value is invalid. + + """ + if value in ["UNKNOWN", "FUTURE"]: + # A valid non-date value. + return None + + # Raises `ValueError` if parse fails. + datetime.datetime.strptime(value, ChangeLogEntry.date_format) + + @classmethod + def validate_maintainer(cls, value): + """ Validate the `maintainer` value. + + :param value: The prospective `maintainer` value. + :return: ``None`` if the value is valid. + :raises ValueError: If the value is invalid. + + """ + valid = False + + if value is None: + valid = True + elif rfc822_person_regex.search(value): + valid = True + + if not valid: + raise ValueError("Not a valid person specification {value!r}") + else: + return None + + @classmethod + def make_ordered_dict(cls, fields): + """ Make an ordered dict of the fields. """ + result = collections.OrderedDict( + (name, fields[name]) + for name in cls.field_names) + return result + + def as_version_info_entry(self): + """ Format the changelog entry as a version info entry. """ + fields = vars(self) + entry = self.make_ordered_dict(fields) + + return entry + + +class InvalidFormatError(ValueError): + """ Raised when the document is not a valid ‘ChangeLog’ document. """ + + +class VersionInfoTranslator(object): + """ Translator from document nodes to a version info stream. """ + + # This class needs its base class to be a class from `docutils`. + # But that would create a circular dependency: Setuptools cannot + # ensure `docutils` is available before importing this module. + # + # Use `ensure_class_bases_begin_with` after importing `docutils`, + # to re-bind the `VersionInfoTranslator` name to a new type that + # inherits from `docutils.nodes.SparseNodeVisitor`. + + __metaclass__ = type + + wrap_width = 78 + bullet_text = "* " + + attr_convert_funcs_by_attr_name = { + 'released': ('release_date', unicode), + 'version': ('version', unicode), + 'maintainer': ('maintainer', unicode), + } + + def __init__(self, document): + super(VersionInfoTranslator, self).__init__(document) + self.settings = document.settings + self.current_section_level = 0 + self.current_field_name = None + self.content = [] + self.indent_width = 0 + self.initial_indent = "" + self.subsequent_indent = "" + self.current_entry = None + + # Docutils is not available when this class is defined. + # Get the `docutils` module dynamically. + self._docutils = sys.modules['docutils'] + + def astext(self): + """ Return the translated document as text. """ + text = json.dumps(self.content, indent=4) + return text + + def append_to_current_entry(self, text): + if self.current_entry is not None: + if self.current_entry.body is not None: + self.current_entry.body += text + + def visit_Text(self, node): + raw_text = node.astext() + text = textwrap.fill( + raw_text, + width=self.wrap_width, + initial_indent=self.initial_indent, + subsequent_indent=self.subsequent_indent) + self.append_to_current_entry(text) + + def depart_Text(self, node): + pass + + def visit_comment(self, node): + raise self._docutils.nodes.SkipNode + + def visit_field_body(self, node): + field_list_node = node.parent.parent + if not isinstance(field_list_node, self._docutils.nodes.field_list): + raise InvalidFormatError( + "Unexpected field within {node!r}".format( + node=field_list_node)) + (attr_name, convert_func) = self.attr_convert_funcs_by_attr_name[ + self.current_field_name] + attr_value = convert_func(node.astext()) + setattr(self.current_entry, attr_name, attr_value) + + def depart_field_body(self, node): + pass + + def visit_field_list(self, node): + pass + + def depart_field_list(self, node): + self.current_field_name = None + self.current_entry.body = "" + + def visit_field_name(self, node): + field_name = node.astext() + if self.current_section_level == 1: + # At a top-level section. + if field_name.lower() not in ["released", "maintainer"]: + raise InvalidFormatError( + "Unexpected field name {name!r}".format(name=field_name)) + self.current_field_name = field_name.lower() + + def depart_field_name(self, node): + pass + + def visit_bullet_list(self, node): + self.current_context = [] + + def depart_bullet_list(self, node): + self.current_entry.changes = self.current_context + self.current_context = None + + def adjust_indent_width(self, delta): + self.indent_width += delta + self.subsequent_indent = " " * self.indent_width + self.initial_indent = self.subsequent_indent + + def visit_list_item(self, node): + indent_delta = +len(self.bullet_text) + self.adjust_indent_width(indent_delta) + self.initial_indent = self.subsequent_indent[:-indent_delta] + self.append_to_current_entry(self.initial_indent + self.bullet_text) + + def depart_list_item(self, node): + indent_delta = +len(self.bullet_text) + self.adjust_indent_width(-indent_delta) + self.append_to_current_entry("\n") + + def visit_section(self, node): + self.current_section_level += 1 + if self.current_section_level == 1: + # At a top-level section. + self.current_entry = ChangeLogEntry() + else: + raise InvalidFormatError( + "Subsections not implemented for this writer") + + def depart_section(self, node): + self.current_section_level -= 1 + self.content.append( + self.current_entry.as_version_info_entry()) + self.current_entry = None + + _expected_title_word_length = len("Version FOO".split(" ")) + + def depart_title(self, node): + title_text = node.astext() + # At a top-level section. + words = title_text.split(" ") + version = None + if len(words) != self._expected_title_word_length: + raise InvalidFormatError( + "Unexpected title text {text!r}".format(text=title_text)) + if words[0].lower() not in ["version"]: + raise InvalidFormatError( + "Unexpected title text {text!r}".format(text=title_text)) + version = words[-1] + self.current_entry.version = version + + +def changelog_to_version_info_collection(infile): + """ Render the ‘ChangeLog’ document to a version info collection. + + :param infile: A file-like object containing the changelog. + :return: The serialised JSON data of the version info collection. + + """ + + # Docutils is not available when Setuptools needs this module, so + # delay the imports to this function instead. + import docutils.core + import docutils.nodes + import docutils.writers + + ensure_class_bases_begin_with( + globals(), str('VersionInfoWriter'), docutils.writers.Writer) + ensure_class_bases_begin_with( + globals(), str('VersionInfoTranslator'), + docutils.nodes.SparseNodeVisitor) + + writer = VersionInfoWriter() + settings_overrides = { + 'doctitle_xform': False, + } + version_info_json = docutils.core.publish_string( + infile.read(), writer=writer, + settings_overrides=settings_overrides) + + return version_info_json + + +try: + lru_cache = functools.lru_cache +except AttributeError: + # Python < 3.2 does not have the `functools.lru_cache` function. + # Not essential, so replace it with a no-op. + lru_cache = lambda maxsize=None, typed=False: lambda func: func + + +@lru_cache(maxsize=128) +def generate_version_info_from_changelog(infile_path): + """ Get the version info for the latest version in the changelog. + + :param infile_path: Filesystem path to the input changelog file. + :return: The generated version info mapping; or ``None`` if the + file cannot be read. + + The document is explicitly opened as UTF-8 encoded text. + + """ + version_info = collections.OrderedDict() + + versions_all_json = None + try: + with io.open(infile_path, 'rt', encoding="utf-8") as infile: + versions_all_json = changelog_to_version_info_collection(infile) + except EnvironmentError: + # If we can't read the input file, leave the collection empty. + pass + + if versions_all_json is not None: + versions_all = json.loads(versions_all_json.decode('utf-8')) + version_info = get_latest_version(versions_all) + + return version_info + + +def get_latest_version(versions): + """ Get the latest version from a collection of changelog entries. + + :param versions: A collection of mappings for changelog entries. + :return: An ordered mapping of fields for the latest version, + if `versions` is non-empty; otherwise, an empty mapping. + + """ + version_info = collections.OrderedDict() + + versions_by_release_date = { + item['release_date']: item + for item in versions} + if versions_by_release_date: + latest_release_date = max(versions_by_release_date.keys()) + version_info = ChangeLogEntry.make_ordered_dict( + versions_by_release_date[latest_release_date]) + + return version_info + + +def serialise_version_info_from_mapping(version_info): + """ Generate the version info serialised data. + + :param version_info: Mapping of version info items. + :return: The version info serialised to JSON. + + """ + content = json.dumps(version_info, indent=4) + + return content + + +changelog_filename = "ChangeLog" + +def get_changelog_path(distribution, filename=changelog_filename): + """ Get the changelog file path for the distribution. + + :param distribution: The distutils.dist.Distribution instance. + :param filename: The base filename of the changelog document. + :return: Filesystem path of the changelog document, or ``None`` + if not discoverable. + + """ + setup_dirname = os.path.dirname(distribution.script_name) + filepath = os.path.join(setup_dirname, filename) + + return filepath + + +def has_changelog(command): + """ Return ``True`` iff the distribution's changelog file exists. """ + result = False + + changelog_path = get_changelog_path(command.distribution) + if changelog_path is not None: + if os.path.exists(changelog_path): + result = True + + return result + + +class EggInfoCommand(setuptools.command.egg_info.egg_info, object): + """ Custom ‘egg_info’ command for this distribution. """ + + sub_commands = ([ + ('write_version_info', has_changelog), + ] + setuptools.command.egg_info.egg_info.sub_commands) + + def run(self): + """ Execute this command. """ + super(EggInfoCommand, self).run() + + for command_name in self.get_sub_commands(): + self.run_command(command_name) + + +version_info_filename = "version_info.json" + +class WriteVersionInfoCommand(EggInfoCommand, object): + """ Setuptools command to serialise version info metadata. """ + + user_options = ([ + ("changelog-path=", None, + "Filesystem path to the changelog document."), + ("outfile-path=", None, + "Filesystem path to the version info file."), + ] + EggInfoCommand.user_options) + + def initialize_options(self): + """ Initialise command options to defaults. """ + super(WriteVersionInfoCommand, self).initialize_options() + self.changelog_path = None + self.outfile_path = None + + def finalize_options(self): + """ Finalise command options before execution. """ + self.set_undefined_options( + 'build', + ('force', 'force')) + + super(WriteVersionInfoCommand, self).finalize_options() + + if self.changelog_path is None: + self.changelog_path = get_changelog_path(self.distribution) + + if self.outfile_path is None: + egg_dir = self.egg_info + self.outfile_path = os.path.join(egg_dir, version_info_filename) + + def run(self): + """ Execute this command. """ + version_info = generate_version_info_from_changelog(self.changelog_path) + content = serialise_version_info_from_mapping(version_info) + self.write_file("version info", self.outfile_path, content) + + +# Local variables: +# coding: utf-8 +# mode: python +# End: +# vim: fileencoding=utf-8 filetype=python : diff --git a/scripts/external_libs/termstyle/MANIFEST.in b/scripts/external_libs/termstyle/MANIFEST.in new file mode 100644 index 00000000..14dafaf3 --- /dev/null +++ b/scripts/external_libs/termstyle/MANIFEST.in @@ -0,0 +1 @@ +include test* diff --git a/scripts/external_libs/termstyle/Makefile b/scripts/external_libs/termstyle/Makefile new file mode 100644 index 00000000..02151dca --- /dev/null +++ b/scripts/external_libs/termstyle/Makefile @@ -0,0 +1,9 @@ +0: + mkzero-gfxmonk \ + -v `cat VERSION` \ + -p termstyle.py \ + -p setup.py \ + python-termstyle.xml + +pypi: + ./setup.py register sdist upload diff --git a/scripts/external_libs/termstyle/README.rst b/scripts/external_libs/termstyle/README.rst new file mode 100644 index 00000000..f3dfa0ab --- /dev/null +++ b/scripts/external_libs/termstyle/README.rst @@ -0,0 +1,82 @@ +========= +termstyle +========= + +termstyle is a simple python library for adding coloured output to +terminal (console) programs. The definitions come from ECMA-048_, the +"Control Functions for Coded Character Sets" standard. + +Installation: +------------- + +I thoroughly recommend using the zero-install feed (see the project homepage) to manage your dependencies if at all possible. zero-install_ provides a much better system than pip or easy_install, and works with absolutely any language and allows decentralised package management that requires no special privileges to install. + +Example Usage: +-------------- +:: + + from termstyle import * + print "%s:%s" % (red('Hey'), green('how are you?')) + print blue('How ', bold('you'), ' doin?') + +or, you can use a colour just as a string:: + + print "%sBlue!%s" % (blue, reset) + +Styles: +------- +:: + + reset or default (no colour / style) + +colour:: + + black + red + green + yellow + blue + magenta + cyan + white + +background colour:: + + bg_black + bg_red + bg_green + bg_yellow + bg_blue + bg_magenta + bg_cyan + bg_white + bg_default + +In terminals supporting transparency ``bg_default`` is often used to set +the background to transparent [#]_. + +weight:: + + bold + inverted + +style:: + + italic + underscore + +Controls: +--------- +:: + + auto() - sets colouring on only if sys.stdout is a terminal + disabe() - disable colours + enable() - enable colours + +.. [#] Supporting terminals include rxvt-unicode_, and Eterm_. + +.. _ECMA-048: http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-048.pdf +.. _rxvt-unicode: http://software.schmorp.de/ +.. _Eterm: http://www.eterm.org/ +.. _zero-install: http://0install.net/ + diff --git a/scripts/external_libs/termstyle/VERSION b/scripts/external_libs/termstyle/VERSION new file mode 100644 index 00000000..345f8cc0 --- /dev/null +++ b/scripts/external_libs/termstyle/VERSION @@ -0,0 +1 @@ +0.1.10
\ No newline at end of file diff --git a/scripts/external_libs/termstyle/__init__.py b/scripts/external_libs/termstyle/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/scripts/external_libs/termstyle/__init__.py diff --git a/scripts/external_libs/termstyle/python-termstyle.xml b/scripts/external_libs/termstyle/python-termstyle.xml new file mode 100644 index 00000000..b6b08bd7 --- /dev/null +++ b/scripts/external_libs/termstyle/python-termstyle.xml @@ -0,0 +1,183 @@ +<?xml version="1.0"?> +<?xml-stylesheet type='text/xsl' href='interface.xsl'?> +<interface xmlns="http://zero-install.sourceforge.net/2004/injector/interface" uri="http://gfxmonk.net/dist/0install/python-termstyle.xml"> + <name>termstyle</name> + <summary>console colouring for python</summary> + <homepage>http://github.com/gfxmonk/termstyle</homepage> + <description> +========= +termstyle +========= + +termstyle is a simple python library for adding coloured output to +terminal (console) programs. The definitions come from ECMA-048_, the +"Control Functions for Coded Character Sets" standard. + +Installation: +------------- + +I thoroughly recommend using the zero-install feed (see the project homepage) to manage your dependencies if at all possible. zero-install_ provides a much better system than pip or easy_install, and works with absolutely any language and allows decentralised package management that requires no special privileges to install. + +Example Usage: +-------------- +:: + + from termstyle import * + print "%s:%s" % (red('Hey'), green('how are you?')) + print blue('How ', bold('you'), ' doin?') + +or, you can use a colour just as a string:: + + print "%sBlue!%s" % (blue, reset) + +Styles: +------- +:: + + reset or default (no colour / style) + +colour:: + + black + red + green + yellow + blue + magenta + cyan + white + +background colour:: + + bg_black + bg_red + bg_green + bg_yellow + bg_blue + bg_magenta + bg_cyan + bg_white + bg_default + +In terminals supporting transparency ``bg_default`` is often used to set +the background to transparent [#]_. + +weight:: + + bold + inverted + +style:: + + italic + underscore + +Controls: +--------- +:: + + auto() - sets colouring on only if sys.stdout is a terminal + disabe() - disable colours + enable() - enable colours + +.. [#] Supporting terminals include rxvt-unicode_, and Eterm_. + +.. _ECMA-048: http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-048.pdf +.. _rxvt-unicode: http://software.schmorp.de/ +.. _Eterm: http://www.eterm.org/ +.. _zero-install: http://0install.net/ + + </description> + <pypi-extra xmlns="http://gfxmonk.net/dist/0install"><![CDATA[ + classifiers=[ + "License :: OSI Approved :: BSD License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + keywords='output colour console ansi', + license='BSD', + ]]></pypi-extra> + <rich-description xmlns="http://gfxmonk.net/dist/0install"> + <div xmlns="http://www.w3.org/1999/xhtml"> + <h1 id="termstyle">termstyle</h1> + <p>termstyle is a simple python library for adding coloured output to terminal (console) programs. The definitions come from <a href="http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-048.pdf">ECMA-048</a>, the "Control Functions for Coded Character Sets" standard.</p> + <h2 id="installation">Installation:</h2> + <p>I thoroughly recommend using the zero-install feed (see the project homepage) to manage your dependencies if at all possible. <a href="http://0install.net/">zero-install</a> provides a much better system than pip or easy_install, and works with absolutely any language and allows decentralised package management that requires no special privileges to install.</p> + <h2 id="example-usage">Example Usage:</h2> + <pre><code>from termstyle import * +print "%s:%s" % (red('Hey'), green('how are you?')) +print blue('How ', bold('you'), ' doin?') +</code></pre> + <p>or, you can use a colour just as a string:</p> + <pre><code>print "%sBlue!%s" % (blue, reset) +</code></pre> + <h2 id="styles">Styles:</h2> + <pre><code>reset or default (no colour / style) +</code></pre> + <p>colour:</p> + <pre><code>black +red +green +yellow +blue +magenta +cyan +white +</code></pre> + <p>background colour:</p> + <pre><code>bg_black +bg_red +bg_green +bg_yellow +bg_blue +bg_magenta +bg_cyan +bg_white +bg_default +</code></pre> + <p>In terminals supporting transparency <code>bg_default</code> is often used to set the background to transparent [#]_.</p> + <p>weight:</p> + <pre><code>bold +inverted +</code></pre> + <p>style:</p> + <pre><code>italic +underscore +</code></pre> + <h2 id="controls">Controls:</h2> + <pre><code>auto() - sets colouring on only if sys.stdout is a terminal +disabe() - disable colours +enable() - enable colours +</code></pre> + </div> + </rich-description> + <group> + <environment insert="" mode="prepend" name="PYTHONPATH"/> + <implementation id="sha1new=1f5b66dcd48fa38740aa98bb2796f0adb7eeab04" released="2010-07-18" version="0.1.5"> + <manifest-digest sha256="9b0ff9b07c1d424484b5aa20fc2db7827a372d84515e8caf46b005f54a034e07"/> + <archive href="http://gfxmonk.net/dist/0install/python-termstyle/python-termstyle-0.1.2.tgz" size="3114"/> + </implementation> + <implementation id="sha1new=9f728d8ba0eb5379c907b85a9662c54935f1e0ca" released="2010-11-05" version="0.1.6"> + <manifest-digest sha256="a7283c6fe262bc88ed270a91304da243c474d32353c2c9d6c6f4003bacfce27c"/> + <archive href="http://gfxmonk.net/dist/0install/python-termstyle/python-termstyle-0.1.6.tgz" size="3607"/> + </implementation> + <implementation id="sha1new=93d781ad4b723d154462f990254f78d3bab7456e" released="2011-01-29" version="0.1.7"> + <manifest-digest sha256="31488cd005a3f9182e6738068a5e9ddfd6d28db370d8b0a0b16d5bbc53048665"/> + <archive href="http://gfxmonk.net/dist/0install/python-termstyle/python-termstyle-0.1.7.tgz" size="4265"/> + </implementation> + <implementation id="sha1new=ada58698e40a6f11b40dbebaa11f70061b47172d" released="2011-03-15" version="0.1.8"> + <manifest-digest sha256="cbcee0509b194eec74a5fe923999b505293c5d0a20b3d79203fa6f47a942b24e"/> + <archive href="http://gfxmonk.net/dist/0install/python-termstyle/python-termstyle-0.1.8.tgz" size="4312"/> + </implementation> + <implementation id="sha1new=e718d149b78028f3ac39ee4de42e0b7644ad48fb" released="2011-03-16" version="0.1.9"> + <manifest-digest sha256="ee3dc9b097ac0b7682fab8753a5dab45f0a9c189544c13a369ada8b9e1f4fde8"/> + <archive href="http://gfxmonk.net/dist/0install/python-termstyle/python-termstyle-0.1.9.tgz" size="2901"/> + </implementation> + <implementation id="sha1new=0c4adef2eccc7ddcb48264c4bc2d84fcfa1072a4" released="2011-04-30" version="0.1.10"> + <manifest-digest sha256="df3b9e11077995f515239eb8382acdfa1741c35e867916917cf70ce53e49bd59"/> + <archive href="http://gfxmonk.net/dist/0install/python-termstyle/python-termstyle-0.1.10.tgz" size="2903"/> + </implementation> + </group> +</interface> diff --git a/scripts/external_libs/termstyle/setup.py b/scripts/external_libs/termstyle/setup.py new file mode 100644 index 00000000..69b11cbb --- /dev/null +++ b/scripts/external_libs/termstyle/setup.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +## NOTE: ## +## this setup.py was generated by zero2pypi: +## http://gfxmonk.net/dist/0install/zero2pypi.xml + +from setuptools import * +setup( + packages = find_packages(exclude=['test', 'test.*']), + install_requires=['setuptools'], + version='0.1.10', + url='http://gfxmonk.net/dist/0install/python-termstyle.xml', + description='console colouring for python', + long_description='\n**Note**: This package has been built automatically by\n`zero2pypi <http://gfxmonk.net/dist/0install/zero2pypi.xml>`_.\nIf possible, you should use the zero-install feed instead:\nhttp://gfxmonk.net/dist/0install/python-termstyle.xml\n\n----------------\n\n=========\ntermstyle\n=========\n\ntermstyle is a simple python library for adding coloured output to\nterminal (console) programs. The definitions come from ECMA-048_, the\n"Control Functions for Coded Character Sets" standard.\n\nInstallation:\n-------------\n\nI thoroughly recommend using the zero-install feed (see the project homepage) to manage your dependencies if at all possible. zero-install_ provides a much better system than pip or easy_install, and works with absolutely any language and allows decentralised package management that requires no special privileges to install.\n\nExample Usage:\n--------------\n::\n\n\tfrom termstyle import *\n\tprint "%s:%s" % (red(\'Hey\'), green(\'how are you?\'))\n\tprint blue(\'How \', bold(\'you\'), \' doin?\')\n\nor, you can use a colour just as a string::\n\n\tprint "%sBlue!%s" % (blue, reset)\n\nStyles:\n-------\n::\n\n\treset or default (no colour / style)\n\ncolour::\n\n\tblack\n\tred\n\tgreen\n\tyellow\n\tblue\n\tmagenta\n\tcyan\n\twhite\n\nbackground colour::\n\n\tbg_black\n\tbg_red\n\tbg_green\n\tbg_yellow\n\tbg_blue\n\tbg_magenta\n\tbg_cyan\n\tbg_white\n\tbg_default\n\nIn terminals supporting transparency ``bg_default`` is often used to set\nthe background to transparent [#]_.\n\nweight::\n\n\tbold\n\tinverted\n\nstyle::\n\n\titalic\n\tunderscore\n\nControls:\n---------\n::\n\n\tauto() - sets colouring on only if sys.stdout is a terminal\n\tdisabe() - disable colours\n\tenable() - enable colours\n\n.. [#] Supporting terminals include rxvt-unicode_, and Eterm_.\n\n.. _ECMA-048: http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-048.pdf\n.. _rxvt-unicode: http://software.schmorp.de/\n.. _Eterm: http://www.eterm.org/\n.. _zero-install: http://0install.net/\n', + name='python-termstyle', + download_url='http://gfxmonk.net/dist/0install/python-termstyle/python-termstyle-0.1.10.tgz', + py_modules=['termstyle'], +classifiers=[ + "License :: OSI Approved :: BSD License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + keywords='output colour console ansi', + license='BSD', +) diff --git a/scripts/external_libs/termstyle/termstyle.py b/scripts/external_libs/termstyle/termstyle.py new file mode 100644 index 00000000..62a3a920 --- /dev/null +++ b/scripts/external_libs/termstyle/termstyle.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# Copyright (c) 2009, Tim Cuthbertson +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of the organisation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import sys + +class Style(object): + prefix='\x1b[' + suffix='m' + enabled = True + + def __init__(self, on_codes, off_codes = 0): + self._on = self.sequence(on_codes) + self._off = self.sequence(off_codes) + + def _get_on(self): return self._on if self.enabled else '' + def _get_off(self): return self._off if self.enabled else '' + on = property(_get_on) + off = property(_get_off) + + @classmethod + def sequence(cls, codes): + wrap_single = lambda code: "%s%s%s" % (cls.prefix, code, cls.suffix) + try: + return ''.join([wrap_single(code) for code in codes]) + except TypeError: + return wrap_single(codes) + + def __str__(self): + if not self.enabled: + return '' + return self.on + + def __call__(self, *args): + contents = ''.join(["%s%s" % (self.on, arg) for arg in args]) + return "%s%s" % (contents, self.off) + + +def auto(): + """set colouring on if STDOUT is a terminal device, off otherwise""" + try: + Style.enabled = False + Style.enabled = sys.stdout.isatty() + except (AttributeError, TypeError): + pass + +def enable(): + """force coloured output""" + Style.enabled = True + +def disable(): + """disable coloured output""" + Style.enabled = False + +default = reset = Style(0) + +black = Style(30) +red = Style(31) +green = Style(32) +yellow = Style(33) +blue = Style(34) +magenta = Style(35) +cyan = Style(36) +white = Style(37) + +bg_black = Style(40) +bg_red = Style(41) +bg_green = Style(42) +bg_yellow = Style(43) +bg_blue = Style(44) +bg_magenta = Style(45) +bg_cyan = Style(46) +bg_white = Style(47) +bg_default = Style(49) + +bold = Style(1) +underscore = Style(4) +inverted = Style(7) +italic = Style(3) + diff --git a/scripts/external_libs/termstyle/test2.py b/scripts/external_libs/termstyle/test2.py new file mode 100644 index 00000000..2d84c375 --- /dev/null +++ b/scripts/external_libs/termstyle/test2.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python2 + +from termstyle import * + +print green(u"unicod\xe9!") diff --git a/scripts/external_libs/termstyle/test3.py b/scripts/external_libs/termstyle/test3.py new file mode 100644 index 00000000..861c44f9 --- /dev/null +++ b/scripts/external_libs/termstyle/test3.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +from termstyle import * + +print(green("unicod\xe9!")) diff --git a/scripts/external_libs/termstyle/test_all.sh b/scripts/external_libs/termstyle/test_all.sh new file mode 100644 index 00000000..d28545a9 --- /dev/null +++ b/scripts/external_libs/termstyle/test_all.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -ex +./test2.py +./test3.py diff --git a/scripts/external_libs/zmq/__init__.py b/scripts/external_libs/zmq/__init__.py new file mode 100644 index 00000000..3408b3ba --- /dev/null +++ b/scripts/external_libs/zmq/__init__.py @@ -0,0 +1,64 @@ +"""Python bindings for 0MQ.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import os +import sys +import glob + +# load bundled libzmq, if there is one: + +here = os.path.dirname(__file__) + +bundled = [] +bundled_sodium = [] +for ext in ('pyd', 'so', 'dll', 'dylib'): + bundled_sodium.extend(glob.glob(os.path.join(here, 'libsodium*.%s*' % ext))) + bundled.extend(glob.glob(os.path.join(here, 'libzmq*.%s*' % ext))) + +if bundled: + import ctypes + if bundled_sodium: + if bundled[0].endswith('.pyd'): + # a Windows Extension + _libsodium = ctypes.cdll.LoadLibrary(bundled_sodium[0]) + else: + _libsodium = ctypes.CDLL(bundled_sodium[0], mode=ctypes.RTLD_GLOBAL) + if bundled[0].endswith('.pyd'): + # a Windows Extension + _libzmq = ctypes.cdll.LoadLibrary(bundled[0]) + else: + _libzmq = ctypes.CDLL(bundled[0], mode=ctypes.RTLD_GLOBAL) + del ctypes +else: + import zipimport + try: + if isinstance(__loader__, zipimport.zipimporter): + # a zipped pyzmq egg + from zmq import libzmq as _libzmq + except (NameError, ImportError): + pass + finally: + del zipimport + +del os, sys, glob, here, bundled, bundled_sodium, ext + +# zmq top-level imports + +from zmq import backend +from zmq.backend import * +from zmq import sugar +from zmq.sugar import * +from zmq import devices + +def get_includes(): + """Return a list of directories to include for linking against pyzmq with cython.""" + from os.path import join, dirname, abspath, pardir + base = dirname(__file__) + parent = abspath(join(base, pardir)) + return [ parent ] + [ join(parent, base, subdir) for subdir in ('utils',) ] + + +__all__ = ['get_includes'] + sugar.__all__ + backend.__all__ + diff --git a/scripts/external_libs/zmq/auth/__init__.py b/scripts/external_libs/zmq/auth/__init__.py new file mode 100644 index 00000000..11d3ad6b --- /dev/null +++ b/scripts/external_libs/zmq/auth/__init__.py @@ -0,0 +1,10 @@ +"""Utilities for ZAP authentication. + +To run authentication in a background thread, see :mod:`zmq.auth.thread`. +For integration with the tornado eventloop, see :mod:`zmq.auth.ioloop`. + +.. versionadded:: 14.1 +""" + +from .base import * +from .certs import * diff --git a/scripts/external_libs/zmq/auth/base.py b/scripts/external_libs/zmq/auth/base.py new file mode 100644 index 00000000..9b4aaed7 --- /dev/null +++ b/scripts/external_libs/zmq/auth/base.py @@ -0,0 +1,272 @@ +"""Base implementation of 0MQ authentication.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import logging + +import zmq +from zmq.utils import z85 +from zmq.utils.strtypes import bytes, unicode, b, u +from zmq.error import _check_version + +from .certs import load_certificates + + +CURVE_ALLOW_ANY = '*' +VERSION = b'1.0' + +class Authenticator(object): + """Implementation of ZAP authentication for zmq connections. + + Note: + - libzmq provides four levels of security: default NULL (which the Authenticator does + not see), and authenticated NULL, PLAIN, and CURVE, which the Authenticator can see. + - until you add policies, all incoming NULL connections are allowed + (classic ZeroMQ behavior), and all PLAIN and CURVE connections are denied. + """ + + def __init__(self, context=None, encoding='utf-8', log=None): + _check_version((4,0), "security") + self.context = context or zmq.Context.instance() + self.encoding = encoding + self.allow_any = False + self.zap_socket = None + self.whitelist = set() + self.blacklist = set() + # passwords is a dict keyed by domain and contains values + # of dicts with username:password pairs. + self.passwords = {} + # certs is dict keyed by domain and contains values + # of dicts keyed by the public keys from the specified location. + self.certs = {} + self.log = log or logging.getLogger('zmq.auth') + + def start(self): + """Create and bind the ZAP socket""" + self.zap_socket = self.context.socket(zmq.REP) + self.zap_socket.linger = 1 + self.zap_socket.bind("inproc://zeromq.zap.01") + + def stop(self): + """Close the ZAP socket""" + if self.zap_socket: + self.zap_socket.close() + self.zap_socket = None + + def allow(self, *addresses): + """Allow (whitelist) IP address(es). + + Connections from addresses not in the whitelist will be rejected. + + - For NULL, all clients from this address will be accepted. + - For PLAIN and CURVE, they will be allowed to continue with authentication. + + whitelist is mutually exclusive with blacklist. + """ + if self.blacklist: + raise ValueError("Only use a whitelist or a blacklist, not both") + self.whitelist.update(addresses) + + def deny(self, *addresses): + """Deny (blacklist) IP address(es). + + Addresses not in the blacklist will be allowed to continue with authentication. + + Blacklist is mutually exclusive with whitelist. + """ + if self.whitelist: + raise ValueError("Only use a whitelist or a blacklist, not both") + self.blacklist.update(addresses) + + def configure_plain(self, domain='*', passwords=None): + """Configure PLAIN authentication for a given domain. + + PLAIN authentication uses a plain-text password file. + To cover all domains, use "*". + You can modify the password file at any time; it is reloaded automatically. + """ + if passwords: + self.passwords[domain] = passwords + + def configure_curve(self, domain='*', location=None): + """Configure CURVE authentication for a given domain. + + CURVE authentication uses a directory that holds all public client certificates, + i.e. their public keys. + + To cover all domains, use "*". + + You can add and remove certificates in that directory at any time. + + To allow all client keys without checking, specify CURVE_ALLOW_ANY for the location. + """ + # If location is CURVE_ALLOW_ANY then allow all clients. Otherwise + # treat location as a directory that holds the certificates. + if location == CURVE_ALLOW_ANY: + self.allow_any = True + else: + self.allow_any = False + try: + self.certs[domain] = load_certificates(location) + except Exception as e: + self.log.error("Failed to load CURVE certs from %s: %s", location, e) + + def handle_zap_message(self, msg): + """Perform ZAP authentication""" + if len(msg) < 6: + self.log.error("Invalid ZAP message, not enough frames: %r", msg) + if len(msg) < 2: + self.log.error("Not enough information to reply") + else: + self._send_zap_reply(msg[1], b"400", b"Not enough frames") + return + + version, request_id, domain, address, identity, mechanism = msg[:6] + credentials = msg[6:] + + domain = u(domain, self.encoding, 'replace') + address = u(address, self.encoding, 'replace') + + if (version != VERSION): + self.log.error("Invalid ZAP version: %r", msg) + self._send_zap_reply(request_id, b"400", b"Invalid version") + return + + self.log.debug("version: %r, request_id: %r, domain: %r," + " address: %r, identity: %r, mechanism: %r", + version, request_id, domain, + address, identity, mechanism, + ) + + + # Is address is explicitly whitelisted or blacklisted? + allowed = False + denied = False + reason = b"NO ACCESS" + + if self.whitelist: + if address in self.whitelist: + allowed = True + self.log.debug("PASSED (whitelist) address=%s", address) + else: + denied = True + reason = b"Address not in whitelist" + self.log.debug("DENIED (not in whitelist) address=%s", address) + + elif self.blacklist: + if address in self.blacklist: + denied = True + reason = b"Address is blacklisted" + self.log.debug("DENIED (blacklist) address=%s", address) + else: + allowed = True + self.log.debug("PASSED (not in blacklist) address=%s", address) + + # Perform authentication mechanism-specific checks if necessary + username = u("user") + if not denied: + + if mechanism == b'NULL' and not allowed: + # For NULL, we allow if the address wasn't blacklisted + self.log.debug("ALLOWED (NULL)") + allowed = True + + elif mechanism == b'PLAIN': + # For PLAIN, even a whitelisted address must authenticate + if len(credentials) != 2: + self.log.error("Invalid PLAIN credentials: %r", credentials) + self._send_zap_reply(request_id, b"400", b"Invalid credentials") + return + username, password = [ u(c, self.encoding, 'replace') for c in credentials ] + allowed, reason = self._authenticate_plain(domain, username, password) + + elif mechanism == b'CURVE': + # For CURVE, even a whitelisted address must authenticate + if len(credentials) != 1: + self.log.error("Invalid CURVE credentials: %r", credentials) + self._send_zap_reply(request_id, b"400", b"Invalid credentials") + return + key = credentials[0] + allowed, reason = self._authenticate_curve(domain, key) + + if allowed: + self._send_zap_reply(request_id, b"200", b"OK", username) + else: + self._send_zap_reply(request_id, b"400", reason) + + def _authenticate_plain(self, domain, username, password): + """PLAIN ZAP authentication""" + allowed = False + reason = b"" + if self.passwords: + # If no domain is not specified then use the default domain + if not domain: + domain = '*' + + if domain in self.passwords: + if username in self.passwords[domain]: + if password == self.passwords[domain][username]: + allowed = True + else: + reason = b"Invalid password" + else: + reason = b"Invalid username" + else: + reason = b"Invalid domain" + + if allowed: + self.log.debug("ALLOWED (PLAIN) domain=%s username=%s password=%s", + domain, username, password, + ) + else: + self.log.debug("DENIED %s", reason) + + else: + reason = b"No passwords defined" + self.log.debug("DENIED (PLAIN) %s", reason) + + return allowed, reason + + def _authenticate_curve(self, domain, client_key): + """CURVE ZAP authentication""" + allowed = False + reason = b"" + if self.allow_any: + allowed = True + reason = b"OK" + self.log.debug("ALLOWED (CURVE allow any client)") + else: + # If no explicit domain is specified then use the default domain + if not domain: + domain = '*' + + if domain in self.certs: + # The certs dict stores keys in z85 format, convert binary key to z85 bytes + z85_client_key = z85.encode(client_key) + if z85_client_key in self.certs[domain] or self.certs[domain] == b'OK': + allowed = True + reason = b"OK" + else: + reason = b"Unknown key" + + status = "ALLOWED" if allowed else "DENIED" + self.log.debug("%s (CURVE) domain=%s client_key=%s", + status, domain, z85_client_key, + ) + else: + reason = b"Unknown domain" + + return allowed, reason + + def _send_zap_reply(self, request_id, status_code, status_text, user_id='user'): + """Send a ZAP reply to finish the authentication.""" + user_id = user_id if status_code == b'200' else b'' + if isinstance(user_id, unicode): + user_id = user_id.encode(self.encoding, 'replace') + metadata = b'' # not currently used + self.log.debug("ZAP reply code=%s text=%s", status_code, status_text) + reply = [VERSION, request_id, status_code, status_text, user_id, metadata] + self.zap_socket.send_multipart(reply) + +__all__ = ['Authenticator', 'CURVE_ALLOW_ANY'] diff --git a/scripts/external_libs/zmq/auth/certs.py b/scripts/external_libs/zmq/auth/certs.py new file mode 100644 index 00000000..4d26ad7b --- /dev/null +++ b/scripts/external_libs/zmq/auth/certs.py @@ -0,0 +1,119 @@ +"""0MQ authentication related functions and classes.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import datetime +import glob +import io +import os +import zmq +from zmq.utils.strtypes import bytes, unicode, b, u + + +_cert_secret_banner = u("""# **** Generated on {0} by pyzmq **** +# ZeroMQ CURVE **Secret** Certificate +# DO NOT PROVIDE THIS FILE TO OTHER USERS nor change its permissions. + +""") + +_cert_public_banner = u("""# **** Generated on {0} by pyzmq **** +# ZeroMQ CURVE Public Certificate +# Exchange securely, or use a secure mechanism to verify the contents +# of this file after exchange. Store public certificates in your home +# directory, in the .curve subdirectory. + +""") + +def _write_key_file(key_filename, banner, public_key, secret_key=None, metadata=None, encoding='utf-8'): + """Create a certificate file""" + if isinstance(public_key, bytes): + public_key = public_key.decode(encoding) + if isinstance(secret_key, bytes): + secret_key = secret_key.decode(encoding) + with io.open(key_filename, 'w', encoding='utf8') as f: + f.write(banner.format(datetime.datetime.now())) + + f.write(u('metadata\n')) + if metadata: + for k, v in metadata.items(): + if isinstance(v, bytes): + v = v.decode(encoding) + f.write(u(" {0} = {1}\n").format(k, v)) + + f.write(u('curve\n')) + f.write(u(" public-key = \"{0}\"\n").format(public_key)) + + if secret_key: + f.write(u(" secret-key = \"{0}\"\n").format(secret_key)) + + +def create_certificates(key_dir, name, metadata=None): + """Create zmq certificates. + + Returns the file paths to the public and secret certificate files. + """ + public_key, secret_key = zmq.curve_keypair() + base_filename = os.path.join(key_dir, name) + secret_key_file = "{0}.key_secret".format(base_filename) + public_key_file = "{0}.key".format(base_filename) + now = datetime.datetime.now() + + _write_key_file(public_key_file, + _cert_public_banner.format(now), + public_key) + + _write_key_file(secret_key_file, + _cert_secret_banner.format(now), + public_key, + secret_key=secret_key, + metadata=metadata) + + return public_key_file, secret_key_file + + +def load_certificate(filename): + """Load public and secret key from a zmq certificate. + + Returns (public_key, secret_key) + + If the certificate file only contains the public key, + secret_key will be None. + """ + public_key = None + secret_key = None + if not os.path.exists(filename): + raise IOError("Invalid certificate file: {0}".format(filename)) + + with open(filename, 'rb') as f: + for line in f: + line = line.strip() + if line.startswith(b'#'): + continue + if line.startswith(b'public-key'): + public_key = line.split(b"=", 1)[1].strip(b' \t\'"') + if line.startswith(b'secret-key'): + secret_key = line.split(b"=", 1)[1].strip(b' \t\'"') + if public_key and secret_key: + break + + return public_key, secret_key + + +def load_certificates(directory='.'): + """Load public keys from all certificates in a directory""" + certs = {} + if not os.path.isdir(directory): + raise IOError("Invalid certificate directory: {0}".format(directory)) + # Follow czmq pattern of public keys stored in *.key files. + glob_string = os.path.join(directory, "*.key") + + cert_files = glob.glob(glob_string) + for cert_file in cert_files: + public_key, _ = load_certificate(cert_file) + if public_key: + certs[public_key] = 'OK' + return certs + +__all__ = ['create_certificates', 'load_certificate', 'load_certificates'] diff --git a/scripts/external_libs/zmq/auth/ioloop.py b/scripts/external_libs/zmq/auth/ioloop.py new file mode 100644 index 00000000..1f448b47 --- /dev/null +++ b/scripts/external_libs/zmq/auth/ioloop.py @@ -0,0 +1,34 @@ +"""ZAP Authenticator integrated with the tornado IOLoop. + +.. versionadded:: 14.1 +""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from zmq.eventloop import ioloop, zmqstream +from .base import Authenticator + + +class IOLoopAuthenticator(Authenticator): + """ZAP authentication for use in the tornado IOLoop""" + + def __init__(self, context=None, encoding='utf-8', log=None, io_loop=None): + super(IOLoopAuthenticator, self).__init__(context) + self.zap_stream = None + self.io_loop = io_loop or ioloop.IOLoop.instance() + + def start(self): + """Start ZAP authentication""" + super(IOLoopAuthenticator, self).start() + self.zap_stream = zmqstream.ZMQStream(self.zap_socket, self.io_loop) + self.zap_stream.on_recv(self.handle_zap_message) + + def stop(self): + """Stop ZAP authentication""" + if self.zap_stream: + self.zap_stream.close() + self.zap_stream = None + super(IOLoopAuthenticator, self).stop() + +__all__ = ['IOLoopAuthenticator'] diff --git a/scripts/external_libs/zmq/auth/thread.py b/scripts/external_libs/zmq/auth/thread.py new file mode 100644 index 00000000..8c3355a9 --- /dev/null +++ b/scripts/external_libs/zmq/auth/thread.py @@ -0,0 +1,184 @@ +"""ZAP Authenticator in a Python Thread. + +.. versionadded:: 14.1 +""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import logging +from threading import Thread + +import zmq +from zmq.utils import jsonapi +from zmq.utils.strtypes import bytes, unicode, b, u + +from .base import Authenticator + +class AuthenticationThread(Thread): + """A Thread for running a zmq Authenticator + + This is run in the background by ThreadedAuthenticator + """ + + def __init__(self, context, endpoint, encoding='utf-8', log=None): + super(AuthenticationThread, self).__init__() + self.context = context or zmq.Context.instance() + self.encoding = encoding + self.log = log = log or logging.getLogger('zmq.auth') + self.authenticator = Authenticator(context, encoding=encoding, log=log) + + # create a socket to communicate back to main thread. + self.pipe = context.socket(zmq.PAIR) + self.pipe.linger = 1 + self.pipe.connect(endpoint) + + def run(self): + """ Start the Authentication Agent thread task """ + self.authenticator.start() + zap = self.authenticator.zap_socket + poller = zmq.Poller() + poller.register(self.pipe, zmq.POLLIN) + poller.register(zap, zmq.POLLIN) + while True: + try: + socks = dict(poller.poll()) + except zmq.ZMQError: + break # interrupted + + if self.pipe in socks and socks[self.pipe] == zmq.POLLIN: + terminate = self._handle_pipe() + if terminate: + break + + if zap in socks and socks[zap] == zmq.POLLIN: + self._handle_zap() + + self.pipe.close() + self.authenticator.stop() + + def _handle_zap(self): + """ + Handle a message from the ZAP socket. + """ + msg = self.authenticator.zap_socket.recv_multipart() + if not msg: return + self.authenticator.handle_zap_message(msg) + + def _handle_pipe(self): + """ + Handle a message from front-end API. + """ + terminate = False + + # Get the whole message off the pipe in one go + msg = self.pipe.recv_multipart() + + if msg is None: + terminate = True + return terminate + + command = msg[0] + self.log.debug("auth received API command %r", command) + + if command == b'ALLOW': + addresses = [u(m, self.encoding) for m in msg[1:]] + try: + self.authenticator.allow(*addresses) + except Exception as e: + self.log.exception("Failed to allow %s", addresses) + + elif command == b'DENY': + addresses = [u(m, self.encoding) for m in msg[1:]] + try: + self.authenticator.deny(*addresses) + except Exception as e: + self.log.exception("Failed to deny %s", addresses) + + elif command == b'PLAIN': + domain = u(msg[1], self.encoding) + json_passwords = msg[2] + self.authenticator.configure_plain(domain, jsonapi.loads(json_passwords)) + + elif command == b'CURVE': + # For now we don't do anything with domains + domain = u(msg[1], self.encoding) + + # If location is CURVE_ALLOW_ANY, allow all clients. Otherwise + # treat location as a directory that holds the certificates. + location = u(msg[2], self.encoding) + self.authenticator.configure_curve(domain, location) + + elif command == b'TERMINATE': + terminate = True + + else: + self.log.error("Invalid auth command from API: %r", command) + + return terminate + +def _inherit_docstrings(cls): + """inherit docstrings from Authenticator, so we don't duplicate them""" + for name, method in cls.__dict__.items(): + if name.startswith('_'): + continue + upstream_method = getattr(Authenticator, name, None) + if not method.__doc__: + method.__doc__ = upstream_method.__doc__ + return cls + +@_inherit_docstrings +class ThreadAuthenticator(object): + """Run ZAP authentication in a background thread""" + + def __init__(self, context=None, encoding='utf-8', log=None): + self.context = context or zmq.Context.instance() + self.log = log + self.encoding = encoding + self.pipe = None + self.pipe_endpoint = "inproc://{0}.inproc".format(id(self)) + self.thread = None + + def allow(self, *addresses): + self.pipe.send_multipart([b'ALLOW'] + [b(a, self.encoding) for a in addresses]) + + def deny(self, *addresses): + self.pipe.send_multipart([b'DENY'] + [b(a, self.encoding) for a in addresses]) + + def configure_plain(self, domain='*', passwords=None): + self.pipe.send_multipart([b'PLAIN', b(domain, self.encoding), jsonapi.dumps(passwords or {})]) + + def configure_curve(self, domain='*', location=''): + domain = b(domain, self.encoding) + location = b(location, self.encoding) + self.pipe.send_multipart([b'CURVE', domain, location]) + + def start(self): + """Start the authentication thread""" + # create a socket to communicate with auth thread. + self.pipe = self.context.socket(zmq.PAIR) + self.pipe.linger = 1 + self.pipe.bind(self.pipe_endpoint) + self.thread = AuthenticationThread(self.context, self.pipe_endpoint, encoding=self.encoding, log=self.log) + self.thread.start() + + def stop(self): + """Stop the authentication thread""" + if self.pipe: + self.pipe.send(b'TERMINATE') + if self.is_alive(): + self.thread.join() + self.thread = None + self.pipe.close() + self.pipe = None + + def is_alive(self): + """Is the ZAP thread currently running?""" + if self.thread and self.thread.is_alive(): + return True + return False + + def __del__(self): + self.stop() + +__all__ = ['ThreadAuthenticator'] diff --git a/scripts/external_libs/zmq/backend/__init__.py b/scripts/external_libs/zmq/backend/__init__.py new file mode 100644 index 00000000..7cac725c --- /dev/null +++ b/scripts/external_libs/zmq/backend/__init__.py @@ -0,0 +1,45 @@ +"""Import basic exposure of libzmq C API as a backend""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import os +import platform +import sys + +from zmq.utils.sixcerpt import reraise + +from .select import public_api, select_backend + +if 'PYZMQ_BACKEND' in os.environ: + backend = os.environ['PYZMQ_BACKEND'] + if backend in ('cython', 'cffi'): + backend = 'zmq.backend.%s' % backend + _ns = select_backend(backend) +else: + # default to cython, fallback to cffi + # (reverse on PyPy) + if platform.python_implementation() == 'PyPy': + first, second = ('zmq.backend.cffi', 'zmq.backend.cython') + else: + first, second = ('zmq.backend.cython', 'zmq.backend.cffi') + + try: + _ns = select_backend(first) + except Exception: + exc_info = sys.exc_info() + exc = exc_info[1] + try: + _ns = select_backend(second) + except ImportError: + # prevent 'During handling of the above exception...' on py3 + # can't use `raise ... from` on Python 2 + if hasattr(exc, '__cause__'): + exc.__cause__ = None + # raise the *first* error, not the fallback + reraise(*exc_info) + +globals().update(_ns) + +__all__ = public_api diff --git a/scripts/external_libs/zmq/backend/cffi/__init__.py b/scripts/external_libs/zmq/backend/cffi/__init__.py new file mode 100644 index 00000000..ca3164d3 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/__init__.py @@ -0,0 +1,22 @@ +"""CFFI backend (for PyPY)""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from zmq.backend.cffi import (constants, error, message, context, socket, + _poll, devices, utils) + +__all__ = [] +for submod in (constants, error, message, context, socket, + _poll, devices, utils): + __all__.extend(submod.__all__) + +from .constants import * +from .error import * +from .message import * +from .context import * +from .socket import * +from .devices import * +from ._poll import * +from ._cffi import zmq_version_info, ffi +from .utils import * diff --git a/scripts/external_libs/zmq/backend/cffi/_cdefs.h b/scripts/external_libs/zmq/backend/cffi/_cdefs.h new file mode 100644 index 00000000..d3300575 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/_cdefs.h @@ -0,0 +1,68 @@ +void zmq_version(int *major, int *minor, int *patch); + +void* zmq_socket(void *context, int type); +int zmq_close(void *socket); + +int zmq_bind(void *socket, const char *endpoint); +int zmq_connect(void *socket, const char *endpoint); + +int zmq_errno(void); +const char * zmq_strerror(int errnum); + +void* zmq_stopwatch_start(void); +unsigned long zmq_stopwatch_stop(void *watch); +void zmq_sleep(int seconds_); +int zmq_device(int device, void *frontend, void *backend); + +int zmq_unbind(void *socket, const char *endpoint); +int zmq_disconnect(void *socket, const char *endpoint); +void* zmq_ctx_new(); +int zmq_ctx_destroy(void *context); +int zmq_ctx_get(void *context, int opt); +int zmq_ctx_set(void *context, int opt, int optval); +int zmq_proxy(void *frontend, void *backend, void *capture); +int zmq_socket_monitor(void *socket, const char *addr, int events); + +int zmq_curve_keypair (char *z85_public_key, char *z85_secret_key); +int zmq_has (const char *capability); + +typedef struct { ...; } zmq_msg_t; +typedef ... zmq_free_fn; + +int zmq_msg_init(zmq_msg_t *msg); +int zmq_msg_init_size(zmq_msg_t *msg, size_t size); +int zmq_msg_init_data(zmq_msg_t *msg, + void *data, + size_t size, + zmq_free_fn *ffn, + void *hint); + +size_t zmq_msg_size(zmq_msg_t *msg); +void *zmq_msg_data(zmq_msg_t *msg); +int zmq_msg_close(zmq_msg_t *msg); + +int zmq_msg_send(zmq_msg_t *msg, void *socket, int flags); +int zmq_msg_recv(zmq_msg_t *msg, void *socket, int flags); + +int zmq_getsockopt(void *socket, + int option_name, + void *option_value, + size_t *option_len); + +int zmq_setsockopt(void *socket, + int option_name, + const void *option_value, + size_t option_len); +typedef struct +{ + void *socket; + int fd; + short events; + short revents; +} zmq_pollitem_t; + +int zmq_poll(zmq_pollitem_t *items, int nitems, long timeout); + +// miscellany +void * memcpy(void *restrict s1, const void *restrict s2, size_t n); +int get_ipc_path_max_len(void); diff --git a/scripts/external_libs/zmq/backend/cffi/_cffi.py b/scripts/external_libs/zmq/backend/cffi/_cffi.py new file mode 100644 index 00000000..c73ebf83 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/_cffi.py @@ -0,0 +1,127 @@ +# coding: utf-8 +"""The main CFFI wrapping of libzmq""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import json +import os +from os.path import dirname, join +from cffi import FFI + +from zmq.utils.constant_names import all_names, no_prefix + + +base_zmq_version = (3,2,2) + +def load_compiler_config(): + """load pyzmq compiler arguments""" + import zmq + zmq_dir = dirname(zmq.__file__) + zmq_parent = dirname(zmq_dir) + + fname = join(zmq_dir, 'utils', 'compiler.json') + if os.path.exists(fname): + with open(fname) as f: + cfg = json.load(f) + else: + cfg = {} + + cfg.setdefault("include_dirs", []) + cfg.setdefault("library_dirs", []) + cfg.setdefault("runtime_library_dirs", []) + cfg.setdefault("libraries", ["zmq"]) + + # cast to str, because cffi can't handle unicode paths (?!) + cfg['libraries'] = [str(lib) for lib in cfg['libraries']] + for key in ("include_dirs", "library_dirs", "runtime_library_dirs"): + # interpret paths relative to parent of zmq (like source tree) + abs_paths = [] + for p in cfg[key]: + if p.startswith('zmq'): + p = join(zmq_parent, p) + abs_paths.append(str(p)) + cfg[key] = abs_paths + return cfg + + +def zmq_version_info(): + """Get libzmq version as tuple of ints""" + major = ffi.new('int*') + minor = ffi.new('int*') + patch = ffi.new('int*') + + C.zmq_version(major, minor, patch) + + return (int(major[0]), int(minor[0]), int(patch[0])) + + +cfg = load_compiler_config() +ffi = FFI() + +def _make_defines(names): + _names = [] + for name in names: + define_line = "#define %s ..." % (name) + _names.append(define_line) + + return "\n".join(_names) + +c_constant_names = [] +for name in all_names: + if no_prefix(name): + c_constant_names.append(name) + else: + c_constant_names.append("ZMQ_" + name) + +# load ffi definitions +here = os.path.dirname(__file__) +with open(os.path.join(here, '_cdefs.h')) as f: + _cdefs = f.read() + +with open(os.path.join(here, '_verify.c')) as f: + _verify = f.read() + +ffi.cdef(_cdefs) +ffi.cdef(_make_defines(c_constant_names)) + +try: + C = ffi.verify(_verify, + modulename='_cffi_ext', + libraries=cfg['libraries'], + include_dirs=cfg['include_dirs'], + library_dirs=cfg['library_dirs'], + runtime_library_dirs=cfg['runtime_library_dirs'], + ) + _version_info = zmq_version_info() +except Exception as e: + raise ImportError("PyZMQ CFFI backend couldn't find zeromq: %s\n" + "Please check that you have zeromq headers and libraries." % e) + +if _version_info < (3,2,2): + raise ImportError("PyZMQ CFFI backend requires zeromq >= 3.2.2," + " but found %i.%i.%i" % _version_info + ) + +nsp = new_sizet_pointer = lambda length: ffi.new('size_t*', length) + +new_uint64_pointer = lambda: (ffi.new('uint64_t*'), + nsp(ffi.sizeof('uint64_t'))) +new_int64_pointer = lambda: (ffi.new('int64_t*'), + nsp(ffi.sizeof('int64_t'))) +new_int_pointer = lambda: (ffi.new('int*'), + nsp(ffi.sizeof('int'))) +new_binary_data = lambda length: (ffi.new('char[%d]' % (length)), + nsp(ffi.sizeof('char') * length)) + +value_uint64_pointer = lambda val : (ffi.new('uint64_t*', val), + ffi.sizeof('uint64_t')) +value_int64_pointer = lambda val: (ffi.new('int64_t*', val), + ffi.sizeof('int64_t')) +value_int_pointer = lambda val: (ffi.new('int*', val), + ffi.sizeof('int')) +value_binary_data = lambda val, length: (ffi.new('char[%d]' % (length + 1), val), + ffi.sizeof('char') * length) + +IPC_PATH_MAX_LEN = C.get_ipc_path_max_len() diff --git a/scripts/external_libs/zmq/backend/cffi/_poll.py b/scripts/external_libs/zmq/backend/cffi/_poll.py new file mode 100644 index 00000000..9bca34ca --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/_poll.py @@ -0,0 +1,56 @@ +# coding: utf-8 +"""zmq poll function""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from ._cffi import C, ffi, zmq_version_info + +from .constants import * + +from zmq.error import _check_rc + + +def _make_zmq_pollitem(socket, flags): + zmq_socket = socket._zmq_socket + zmq_pollitem = ffi.new('zmq_pollitem_t*') + zmq_pollitem.socket = zmq_socket + zmq_pollitem.fd = 0 + zmq_pollitem.events = flags + zmq_pollitem.revents = 0 + return zmq_pollitem[0] + +def _make_zmq_pollitem_fromfd(socket_fd, flags): + zmq_pollitem = ffi.new('zmq_pollitem_t*') + zmq_pollitem.socket = ffi.NULL + zmq_pollitem.fd = socket_fd + zmq_pollitem.events = flags + zmq_pollitem.revents = 0 + return zmq_pollitem[0] + +def zmq_poll(sockets, timeout): + cffi_pollitem_list = [] + low_level_to_socket_obj = {} + for item in sockets: + if isinstance(item[0], int): + low_level_to_socket_obj[item[0]] = item + cffi_pollitem_list.append(_make_zmq_pollitem_fromfd(item[0], item[1])) + else: + low_level_to_socket_obj[item[0]._zmq_socket] = item + cffi_pollitem_list.append(_make_zmq_pollitem(item[0], item[1])) + items = ffi.new('zmq_pollitem_t[]', cffi_pollitem_list) + list_length = ffi.cast('int', len(cffi_pollitem_list)) + c_timeout = ffi.cast('long', timeout) + rc = C.zmq_poll(items, list_length, c_timeout) + _check_rc(rc) + result = [] + for index in range(len(items)): + if not items[index].socket == ffi.NULL: + if items[index].revents > 0: + result.append((low_level_to_socket_obj[items[index].socket][0], + items[index].revents)) + else: + result.append((items[index].fd, items[index].revents)) + return result + +__all__ = ['zmq_poll'] diff --git a/scripts/external_libs/zmq/backend/cffi/_verify.c b/scripts/external_libs/zmq/backend/cffi/_verify.c new file mode 100644 index 00000000..547840eb --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/_verify.c @@ -0,0 +1,12 @@ +#include <stdio.h> +#include <sys/un.h> +#include <string.h> + +#include <zmq.h> +#include <zmq_utils.h> +#include "zmq_compat.h" + +int get_ipc_path_max_len(void) { + struct sockaddr_un *dummy; + return sizeof(dummy->sun_path) - 1; +} diff --git a/scripts/external_libs/zmq/backend/cffi/constants.py b/scripts/external_libs/zmq/backend/cffi/constants.py new file mode 100644 index 00000000..ee293e74 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/constants.py @@ -0,0 +1,15 @@ +# coding: utf-8 +"""zmq constants""" + +from ._cffi import C, c_constant_names +from zmq.utils.constant_names import all_names + +g = globals() +for cname in c_constant_names: + if cname.startswith("ZMQ_"): + name = cname[4:] + else: + name = cname + g[name] = getattr(C, cname) + +__all__ = all_names diff --git a/scripts/external_libs/zmq/backend/cffi/context.py b/scripts/external_libs/zmq/backend/cffi/context.py new file mode 100644 index 00000000..16a7b257 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/context.py @@ -0,0 +1,100 @@ +# coding: utf-8 +"""zmq Context class""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import weakref + +from ._cffi import C, ffi + +from .socket import * +from .constants import * + +from zmq.error import ZMQError, _check_rc + +class Context(object): + _zmq_ctx = None + _iothreads = None + _closed = None + _sockets = None + _shadow = False + + def __init__(self, io_threads=1, shadow=None): + + if shadow: + self._zmq_ctx = ffi.cast("void *", shadow) + self._shadow = True + else: + self._shadow = False + if not io_threads >= 0: + raise ZMQError(EINVAL) + + self._zmq_ctx = C.zmq_ctx_new() + if self._zmq_ctx == ffi.NULL: + raise ZMQError(C.zmq_errno()) + if not shadow: + C.zmq_ctx_set(self._zmq_ctx, IO_THREADS, io_threads) + self._closed = False + self._sockets = set() + + @property + def underlying(self): + """The address of the underlying libzmq context""" + return int(ffi.cast('size_t', self._zmq_ctx)) + + @property + def closed(self): + return self._closed + + def _add_socket(self, socket): + ref = weakref.ref(socket) + self._sockets.add(ref) + return ref + + def _rm_socket(self, ref): + if ref in self._sockets: + self._sockets.remove(ref) + + def set(self, option, value): + """set a context option + + see zmq_ctx_set + """ + rc = C.zmq_ctx_set(self._zmq_ctx, option, value) + _check_rc(rc) + + def get(self, option): + """get context option + + see zmq_ctx_get + """ + rc = C.zmq_ctx_get(self._zmq_ctx, option) + _check_rc(rc) + return rc + + def term(self): + if self.closed: + return + + C.zmq_ctx_destroy(self._zmq_ctx) + + self._zmq_ctx = None + self._closed = True + + def destroy(self, linger=None): + if self.closed: + return + + sockets = self._sockets + self._sockets = set() + for s in sockets: + s = s() + if s and not s.closed: + if linger: + s.setsockopt(LINGER, linger) + s.close() + + self.term() + +__all__ = ['Context'] diff --git a/scripts/external_libs/zmq/backend/cffi/devices.py b/scripts/external_libs/zmq/backend/cffi/devices.py new file mode 100644 index 00000000..c7a514a8 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/devices.py @@ -0,0 +1,24 @@ +# coding: utf-8 +"""zmq device functions""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from ._cffi import C, ffi, zmq_version_info +from .socket import Socket +from zmq.error import ZMQError, _check_rc + +def device(device_type, frontend, backend): + rc = C.zmq_proxy(frontend._zmq_socket, backend._zmq_socket, ffi.NULL) + _check_rc(rc) + +def proxy(frontend, backend, capture=None): + if isinstance(capture, Socket): + capture = capture._zmq_socket + else: + capture = ffi.NULL + + rc = C.zmq_proxy(frontend._zmq_socket, backend._zmq_socket, capture) + _check_rc(rc) + +__all__ = ['device', 'proxy'] diff --git a/scripts/external_libs/zmq/backend/cffi/error.py b/scripts/external_libs/zmq/backend/cffi/error.py new file mode 100644 index 00000000..3bb64de0 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/error.py @@ -0,0 +1,13 @@ +"""zmq error functions""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from ._cffi import C, ffi + +def strerror(errno): + return ffi.string(C.zmq_strerror(errno)) + +zmq_errno = C.zmq_errno + +__all__ = ['strerror', 'zmq_errno'] diff --git a/scripts/external_libs/zmq/backend/cffi/message.py b/scripts/external_libs/zmq/backend/cffi/message.py new file mode 100644 index 00000000..c35decb6 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/message.py @@ -0,0 +1,69 @@ +"""Dummy Frame object""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from ._cffi import ffi, C + +import zmq +from zmq.utils.strtypes import unicode + +try: + view = memoryview +except NameError: + view = buffer + +_content = lambda x: x.tobytes() if type(x) == memoryview else x + +class Frame(object): + _data = None + tracker = None + closed = False + more = False + buffer = None + + + def __init__(self, data, track=False): + try: + view(data) + except TypeError: + raise + + self._data = data + + if isinstance(data, unicode): + raise TypeError("Unicode objects not allowed. Only: str/bytes, " + + "buffer interfaces.") + + self.more = False + self.tracker = None + self.closed = False + if track: + self.tracker = zmq.MessageTracker() + + self.buffer = view(self.bytes) + + @property + def bytes(self): + data = _content(self._data) + return data + + def __len__(self): + return len(self.bytes) + + def __eq__(self, other): + return self.bytes == _content(other) + + def __str__(self): + if str is unicode: + return self.bytes.decode() + else: + return self.bytes + + @property + def done(self): + return True + +Message = Frame + +__all__ = ['Frame', 'Message'] diff --git a/scripts/external_libs/zmq/backend/cffi/socket.py b/scripts/external_libs/zmq/backend/cffi/socket.py new file mode 100644 index 00000000..3c427739 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/socket.py @@ -0,0 +1,244 @@ +# coding: utf-8 +"""zmq Socket class""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import random +import codecs + +import errno as errno_mod + +from ._cffi import (C, ffi, new_uint64_pointer, new_int64_pointer, + new_int_pointer, new_binary_data, value_uint64_pointer, + value_int64_pointer, value_int_pointer, value_binary_data, + IPC_PATH_MAX_LEN) + +from .message import Frame +from .constants import * + +import zmq +from zmq.error import ZMQError, _check_rc, _check_version +from zmq.utils.strtypes import unicode + + +def new_pointer_from_opt(option, length=0): + from zmq.sugar.constants import ( + int64_sockopts, bytes_sockopts, + ) + if option in int64_sockopts: + return new_int64_pointer() + elif option in bytes_sockopts: + return new_binary_data(length) + else: + # default + return new_int_pointer() + +def value_from_opt_pointer(option, opt_pointer, length=0): + from zmq.sugar.constants import ( + int64_sockopts, bytes_sockopts, + ) + if option in int64_sockopts: + return int(opt_pointer[0]) + elif option in bytes_sockopts: + return ffi.buffer(opt_pointer, length)[:] + else: + return int(opt_pointer[0]) + +def initialize_opt_pointer(option, value, length=0): + from zmq.sugar.constants import ( + int64_sockopts, bytes_sockopts, + ) + if option in int64_sockopts: + return value_int64_pointer(value) + elif option in bytes_sockopts: + return value_binary_data(value, length) + else: + return value_int_pointer(value) + + +class Socket(object): + context = None + socket_type = None + _zmq_socket = None + _closed = None + _ref = None + _shadow = False + + def __init__(self, context=None, socket_type=None, shadow=None): + self.context = context + if shadow is not None: + self._zmq_socket = ffi.cast("void *", shadow) + self._shadow = True + else: + self._shadow = False + self._zmq_socket = C.zmq_socket(context._zmq_ctx, socket_type) + if self._zmq_socket == ffi.NULL: + raise ZMQError() + self._closed = False + if context: + self._ref = context._add_socket(self) + + @property + def underlying(self): + """The address of the underlying libzmq socket""" + return int(ffi.cast('size_t', self._zmq_socket)) + + @property + def closed(self): + return self._closed + + def close(self, linger=None): + rc = 0 + if not self._closed and hasattr(self, '_zmq_socket'): + if self._zmq_socket is not None: + rc = C.zmq_close(self._zmq_socket) + self._closed = True + if self.context: + self.context._rm_socket(self._ref) + return rc + + def bind(self, address): + if isinstance(address, unicode): + address = address.encode('utf8') + rc = C.zmq_bind(self._zmq_socket, address) + if rc < 0: + if IPC_PATH_MAX_LEN and C.zmq_errno() == errno_mod.ENAMETOOLONG: + # py3compat: address is bytes, but msg wants str + if str is unicode: + address = address.decode('utf-8', 'replace') + path = address.split('://', 1)[-1] + msg = ('ipc path "{0}" is longer than {1} ' + 'characters (sizeof(sockaddr_un.sun_path)).' + .format(path, IPC_PATH_MAX_LEN)) + raise ZMQError(C.zmq_errno(), msg=msg) + else: + _check_rc(rc) + + def unbind(self, address): + _check_version((3,2), "unbind") + if isinstance(address, unicode): + address = address.encode('utf8') + rc = C.zmq_unbind(self._zmq_socket, address) + _check_rc(rc) + + def connect(self, address): + if isinstance(address, unicode): + address = address.encode('utf8') + rc = C.zmq_connect(self._zmq_socket, address) + _check_rc(rc) + + def disconnect(self, address): + _check_version((3,2), "disconnect") + if isinstance(address, unicode): + address = address.encode('utf8') + rc = C.zmq_disconnect(self._zmq_socket, address) + _check_rc(rc) + + def set(self, option, value): + length = None + if isinstance(value, unicode): + raise TypeError("unicode not allowed, use bytes") + + if isinstance(value, bytes): + if option not in zmq.constants.bytes_sockopts: + raise TypeError("not a bytes sockopt: %s" % option) + length = len(value) + + c_data = initialize_opt_pointer(option, value, length) + + c_value_pointer = c_data[0] + c_sizet = c_data[1] + + rc = C.zmq_setsockopt(self._zmq_socket, + option, + ffi.cast('void*', c_value_pointer), + c_sizet) + _check_rc(rc) + + def get(self, option): + c_data = new_pointer_from_opt(option, length=255) + + c_value_pointer = c_data[0] + c_sizet_pointer = c_data[1] + + rc = C.zmq_getsockopt(self._zmq_socket, + option, + c_value_pointer, + c_sizet_pointer) + _check_rc(rc) + + sz = c_sizet_pointer[0] + v = value_from_opt_pointer(option, c_value_pointer, sz) + if option != zmq.IDENTITY and option in zmq.constants.bytes_sockopts and v.endswith(b'\0'): + v = v[:-1] + return v + + def send(self, message, flags=0, copy=False, track=False): + if isinstance(message, unicode): + raise TypeError("Message must be in bytes, not an unicode Object") + + if isinstance(message, Frame): + message = message.bytes + + zmq_msg = ffi.new('zmq_msg_t*') + c_message = ffi.new('char[]', message) + rc = C.zmq_msg_init_size(zmq_msg, len(message)) + C.memcpy(C.zmq_msg_data(zmq_msg), c_message, len(message)) + + rc = C.zmq_msg_send(zmq_msg, self._zmq_socket, flags) + C.zmq_msg_close(zmq_msg) + _check_rc(rc) + + if track: + return zmq.MessageTracker() + + def recv(self, flags=0, copy=True, track=False): + zmq_msg = ffi.new('zmq_msg_t*') + C.zmq_msg_init(zmq_msg) + + rc = C.zmq_msg_recv(zmq_msg, self._zmq_socket, flags) + + if rc < 0: + C.zmq_msg_close(zmq_msg) + _check_rc(rc) + + _buffer = ffi.buffer(C.zmq_msg_data(zmq_msg), C.zmq_msg_size(zmq_msg)) + value = _buffer[:] + C.zmq_msg_close(zmq_msg) + + frame = Frame(value, track=track) + frame.more = self.getsockopt(RCVMORE) + + if copy: + return frame.bytes + else: + return frame + + def monitor(self, addr, events=-1): + """s.monitor(addr, flags) + + Start publishing socket events on inproc. + See libzmq docs for zmq_monitor for details. + + Note: requires libzmq >= 3.2 + + Parameters + ---------- + addr : str + The inproc url used for monitoring. Passing None as + the addr will cause an existing socket monitor to be + deregistered. + events : int [default: zmq.EVENT_ALL] + The zmq event bitmask for which events will be sent to the monitor. + """ + + _check_version((3,2), "monitor") + if events < 0: + events = zmq.EVENT_ALL + if addr is None: + addr = ffi.NULL + rc = C.zmq_socket_monitor(self._zmq_socket, addr, events) + + +__all__ = ['Socket', 'IPC_PATH_MAX_LEN'] diff --git a/scripts/external_libs/zmq/backend/cffi/utils.py b/scripts/external_libs/zmq/backend/cffi/utils.py new file mode 100644 index 00000000..fde7827b --- /dev/null +++ b/scripts/external_libs/zmq/backend/cffi/utils.py @@ -0,0 +1,62 @@ +# coding: utf-8 +"""miscellaneous zmq_utils wrapping""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from ._cffi import ffi, C + +from zmq.error import ZMQError, _check_rc, _check_version +from zmq.utils.strtypes import unicode + +def has(capability): + """Check for zmq capability by name (e.g. 'ipc', 'curve') + + .. versionadded:: libzmq-4.1 + .. versionadded:: 14.1 + """ + _check_version((4,1), 'zmq.has') + if isinstance(capability, unicode): + capability = capability.encode('utf8') + return bool(C.zmq_has(capability)) + +def curve_keypair(): + """generate a Z85 keypair for use with zmq.CURVE security + + Requires libzmq (≥ 4.0) to have been linked with libsodium. + + Returns + ------- + (public, secret) : two bytestrings + The public and private keypair as 40 byte z85-encoded bytestrings. + """ + _check_version((3,2), "monitor") + public = ffi.new('char[64]') + private = ffi.new('char[64]') + rc = C.zmq_curve_keypair(public, private) + _check_rc(rc) + return ffi.buffer(public)[:40], ffi.buffer(private)[:40] + + +class Stopwatch(object): + def __init__(self): + self.watch = ffi.NULL + + def start(self): + if self.watch == ffi.NULL: + self.watch = C.zmq_stopwatch_start() + else: + raise ZMQError('Stopwatch is already runing.') + + def stop(self): + if self.watch == ffi.NULL: + raise ZMQError('Must start the Stopwatch before calling stop.') + else: + time = C.zmq_stopwatch_stop(self.watch) + self.watch = ffi.NULL + return time + + def sleep(self, seconds): + C.zmq_sleep(seconds) + +__all__ = ['has', 'curve_keypair', 'Stopwatch'] diff --git a/scripts/external_libs/zmq/backend/cython/__init__.py b/scripts/external_libs/zmq/backend/cython/__init__.py new file mode 100644 index 00000000..e5358185 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/__init__.py @@ -0,0 +1,23 @@ +"""Python bindings for core 0MQ objects.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Lesser GNU Public License (LGPL). + +from . import (constants, error, message, context, + socket, utils, _poll, _version, _device ) + +__all__ = [] +for submod in (constants, error, message, context, + socket, utils, _poll, _version, _device): + __all__.extend(submod.__all__) + +from .constants import * +from .error import * +from .message import * +from .context import * +from .socket import * +from ._poll import * +from .utils import * +from ._device import * +from ._version import * + diff --git a/scripts/external_libs/zmq/backend/cython/_device.py b/scripts/external_libs/zmq/backend/cython/_device.py new file mode 100644 index 00000000..3368ca2c --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/_device.py @@ -0,0 +1,7 @@ +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys, pkg_resources, imp + __file__ = pkg_resources.resource_filename(__name__,'_device.so') + __loader__ = None; del __bootstrap__, __loader__ + imp.load_dynamic(__name__,__file__) +__bootstrap__() diff --git a/scripts/external_libs/zmq/backend/cython/_device.so b/scripts/external_libs/zmq/backend/cython/_device.so Binary files differnew file mode 100644 index 00000000..5957f8e0 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/_device.so diff --git a/scripts/external_libs/zmq/backend/cython/_poll.py b/scripts/external_libs/zmq/backend/cython/_poll.py new file mode 100644 index 00000000..cb1d5d77 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/_poll.py @@ -0,0 +1,7 @@ +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys, pkg_resources, imp + __file__ = pkg_resources.resource_filename(__name__,'_poll.so') + __loader__ = None; del __bootstrap__, __loader__ + imp.load_dynamic(__name__,__file__) +__bootstrap__() diff --git a/scripts/external_libs/zmq/backend/cython/_poll.so b/scripts/external_libs/zmq/backend/cython/_poll.so Binary files differnew file mode 100644 index 00000000..d93d0f10 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/_poll.so diff --git a/scripts/external_libs/zmq/backend/cython/_version.py b/scripts/external_libs/zmq/backend/cython/_version.py new file mode 100644 index 00000000..08262706 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/_version.py @@ -0,0 +1,7 @@ +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys, pkg_resources, imp + __file__ = pkg_resources.resource_filename(__name__,'_version.so') + __loader__ = None; del __bootstrap__, __loader__ + imp.load_dynamic(__name__,__file__) +__bootstrap__() diff --git a/scripts/external_libs/zmq/backend/cython/_version.so b/scripts/external_libs/zmq/backend/cython/_version.so Binary files differnew file mode 100644 index 00000000..40dd3dc8 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/_version.so diff --git a/scripts/external_libs/zmq/backend/cython/checkrc.pxd b/scripts/external_libs/zmq/backend/cython/checkrc.pxd new file mode 100644 index 00000000..3bf69fc3 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/checkrc.pxd @@ -0,0 +1,23 @@ +from libc.errno cimport EINTR, EAGAIN +from cpython cimport PyErr_CheckSignals +from libzmq cimport zmq_errno, ZMQ_ETERM + +cdef inline int _check_rc(int rc) except -1: + """internal utility for checking zmq return condition + + and raising the appropriate Exception class + """ + cdef int errno = zmq_errno() + PyErr_CheckSignals() + if rc < 0: + if errno == EAGAIN: + from zmq.error import Again + raise Again(errno) + elif errno == ZMQ_ETERM: + from zmq.error import ContextTerminated + raise ContextTerminated(errno) + else: + from zmq.error import ZMQError + raise ZMQError(errno) + # return -1 + return 0 diff --git a/scripts/external_libs/zmq/backend/cython/constants.py b/scripts/external_libs/zmq/backend/cython/constants.py new file mode 100644 index 00000000..ea772ac0 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/constants.py @@ -0,0 +1,7 @@ +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys, pkg_resources, imp + __file__ = pkg_resources.resource_filename(__name__,'constants.so') + __loader__ = None; del __bootstrap__, __loader__ + imp.load_dynamic(__name__,__file__) +__bootstrap__() diff --git a/scripts/external_libs/zmq/backend/cython/constants.so b/scripts/external_libs/zmq/backend/cython/constants.so Binary files differnew file mode 100644 index 00000000..cf44c07e --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/constants.so diff --git a/scripts/external_libs/zmq/backend/cython/context.pxd b/scripts/external_libs/zmq/backend/cython/context.pxd new file mode 100644 index 00000000..9c9267a5 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/context.pxd @@ -0,0 +1,41 @@ +"""0MQ Context class declaration.""" + +# +# Copyright (c) 2010-2011 Brian E. Granger & Min Ragan-Kelley +# +# This file is part of pyzmq. +# +# pyzmq is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# pyzmq is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- + +cdef class Context: + + cdef object __weakref__ # enable weakref + cdef void *handle # The C handle for the underlying zmq object. + cdef bint _shadow # whether the Context is a shadow wrapper of another + cdef void **_sockets # A C-array containg socket handles + cdef size_t _n_sockets # the number of sockets + cdef size_t _max_sockets # the size of the _sockets array + cdef int _pid # the pid of the process which created me (for fork safety) + + cdef public bint closed # bool property for a closed context. + cdef inline int _term(self) + # helpers for events on _sockets in Socket.__cinit__()/close() + cdef inline void _add_socket(self, void* handle) + cdef inline void _remove_socket(self, void* handle) + diff --git a/scripts/external_libs/zmq/backend/cython/context.py b/scripts/external_libs/zmq/backend/cython/context.py new file mode 100644 index 00000000..19f8ec7c --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/context.py @@ -0,0 +1,7 @@ +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys, pkg_resources, imp + __file__ = pkg_resources.resource_filename(__name__,'context.so') + __loader__ = None; del __bootstrap__, __loader__ + imp.load_dynamic(__name__,__file__) +__bootstrap__() diff --git a/scripts/external_libs/zmq/backend/cython/context.so b/scripts/external_libs/zmq/backend/cython/context.so Binary files differnew file mode 100644 index 00000000..ef9b9699 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/context.so diff --git a/scripts/external_libs/zmq/backend/cython/error.py b/scripts/external_libs/zmq/backend/cython/error.py new file mode 100644 index 00000000..d3a4ea0e --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/error.py @@ -0,0 +1,7 @@ +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys, pkg_resources, imp + __file__ = pkg_resources.resource_filename(__name__,'error.so') + __loader__ = None; del __bootstrap__, __loader__ + imp.load_dynamic(__name__,__file__) +__bootstrap__() diff --git a/scripts/external_libs/zmq/backend/cython/error.so b/scripts/external_libs/zmq/backend/cython/error.so Binary files differnew file mode 100644 index 00000000..360da9dd --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/error.so diff --git a/scripts/external_libs/zmq/backend/cython/libzmq.pxd b/scripts/external_libs/zmq/backend/cython/libzmq.pxd new file mode 100644 index 00000000..e42f6d6b --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/libzmq.pxd @@ -0,0 +1,110 @@ +"""All the C imports for 0MQ""" + +# +# Copyright (c) 2010 Brian E. Granger & Min Ragan-Kelley +# +# This file is part of pyzmq. +# +# pyzmq is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# pyzmq is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Import the C header files +#----------------------------------------------------------------------------- + +cdef extern from *: + ctypedef void* const_void_ptr "const void *" + ctypedef char* const_char_ptr "const char *" + +cdef extern from "zmq_compat.h": + ctypedef signed long long int64_t "pyzmq_int64_t" + +include "constant_enums.pxi" + +cdef extern from "zmq.h" nogil: + + void _zmq_version "zmq_version"(int *major, int *minor, int *patch) + + ctypedef int fd_t "ZMQ_FD_T" + + enum: errno + char *zmq_strerror (int errnum) + int zmq_errno() + + void *zmq_ctx_new () + int zmq_ctx_destroy (void *context) + int zmq_ctx_set (void *context, int option, int optval) + int zmq_ctx_get (void *context, int option) + void *zmq_init (int io_threads) + int zmq_term (void *context) + + # blackbox def for zmq_msg_t + ctypedef void * zmq_msg_t "zmq_msg_t" + + ctypedef void zmq_free_fn(void *data, void *hint) + + int zmq_msg_init (zmq_msg_t *msg) + int zmq_msg_init_size (zmq_msg_t *msg, size_t size) + int zmq_msg_init_data (zmq_msg_t *msg, void *data, + size_t size, zmq_free_fn *ffn, void *hint) + int zmq_msg_send (zmq_msg_t *msg, void *s, int flags) + int zmq_msg_recv (zmq_msg_t *msg, void *s, int flags) + int zmq_msg_close (zmq_msg_t *msg) + int zmq_msg_move (zmq_msg_t *dest, zmq_msg_t *src) + int zmq_msg_copy (zmq_msg_t *dest, zmq_msg_t *src) + void *zmq_msg_data (zmq_msg_t *msg) + size_t zmq_msg_size (zmq_msg_t *msg) + int zmq_msg_more (zmq_msg_t *msg) + int zmq_msg_get (zmq_msg_t *msg, int option) + int zmq_msg_set (zmq_msg_t *msg, int option, int optval) + const_char_ptr zmq_msg_gets (zmq_msg_t *msg, const_char_ptr property) + int zmq_has (const_char_ptr capability) + + void *zmq_socket (void *context, int type) + int zmq_close (void *s) + int zmq_setsockopt (void *s, int option, void *optval, size_t optvallen) + int zmq_getsockopt (void *s, int option, void *optval, size_t *optvallen) + int zmq_bind (void *s, char *addr) + int zmq_connect (void *s, char *addr) + int zmq_unbind (void *s, char *addr) + int zmq_disconnect (void *s, char *addr) + + int zmq_socket_monitor (void *s, char *addr, int flags) + + # send/recv + int zmq_sendbuf (void *s, const_void_ptr buf, size_t n, int flags) + int zmq_recvbuf (void *s, void *buf, size_t n, int flags) + + ctypedef struct zmq_pollitem_t: + void *socket + int fd + short events + short revents + + int zmq_poll (zmq_pollitem_t *items, int nitems, long timeout) + + int zmq_device (int device_, void *insocket_, void *outsocket_) + int zmq_proxy (void *frontend, void *backend, void *capture) + +cdef extern from "zmq_utils.h" nogil: + + void *zmq_stopwatch_start () + unsigned long zmq_stopwatch_stop (void *watch_) + void zmq_sleep (int seconds_) + int zmq_curve_keypair (char *z85_public_key, char *z85_secret_key) + diff --git a/scripts/external_libs/zmq/backend/cython/message.pxd b/scripts/external_libs/zmq/backend/cython/message.pxd new file mode 100644 index 00000000..4781195f --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/message.pxd @@ -0,0 +1,63 @@ +"""0MQ Message related class declarations.""" + +# +# Copyright (c) 2010-2011 Brian E. Granger & Min Ragan-Kelley +# +# This file is part of pyzmq. +# +# pyzmq is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# pyzmq is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from cpython cimport PyBytes_FromStringAndSize + +from libzmq cimport zmq_msg_t, zmq_msg_data, zmq_msg_size + +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- + +cdef class MessageTracker(object): + + cdef set events # Message Event objects to track. + cdef set peers # Other Message or MessageTracker objects. + + +cdef class Frame: + + cdef zmq_msg_t zmq_msg + cdef object _data # The actual message data as a Python object. + cdef object _buffer # A Python Buffer/View of the message contents + cdef object _bytes # A bytes/str copy of the message. + cdef bint _failed_init # Flag to handle failed zmq_msg_init + cdef public object tracker_event # Event for use with zmq_free_fn. + cdef public object tracker # MessageTracker object. + cdef public bint more # whether RCVMORE was set + + cdef Frame fast_copy(self) # Create shallow copy of Message object. + cdef object _getbuffer(self) # Construct self._buffer. + + +cdef inline object copy_zmq_msg_bytes(zmq_msg_t *zmq_msg): + """ Copy the data from a zmq_msg_t """ + cdef char *data_c = NULL + cdef Py_ssize_t data_len_c + data_c = <char *>zmq_msg_data(zmq_msg) + data_len_c = zmq_msg_size(zmq_msg) + return PyBytes_FromStringAndSize(data_c, data_len_c) + + diff --git a/scripts/external_libs/zmq/backend/cython/message.py b/scripts/external_libs/zmq/backend/cython/message.py new file mode 100644 index 00000000..5e423b62 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/message.py @@ -0,0 +1,7 @@ +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys, pkg_resources, imp + __file__ = pkg_resources.resource_filename(__name__,'message.so') + __loader__ = None; del __bootstrap__, __loader__ + imp.load_dynamic(__name__,__file__) +__bootstrap__() diff --git a/scripts/external_libs/zmq/backend/cython/message.so b/scripts/external_libs/zmq/backend/cython/message.so Binary files differnew file mode 100644 index 00000000..f674489f --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/message.so diff --git a/scripts/external_libs/zmq/backend/cython/socket.pxd b/scripts/external_libs/zmq/backend/cython/socket.pxd new file mode 100644 index 00000000..b8a331e2 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/socket.pxd @@ -0,0 +1,47 @@ +"""0MQ Socket class declaration.""" + +# +# Copyright (c) 2010-2011 Brian E. Granger & Min Ragan-Kelley +# +# This file is part of pyzmq. +# +# pyzmq is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# pyzmq is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from context cimport Context + +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- + + +cdef class Socket: + + cdef object __weakref__ # enable weakref + cdef void *handle # The C handle for the underlying zmq object. + cdef bint _shadow # whether the Socket is a shadow wrapper of another + # Hold on to a reference to the context to make sure it is not garbage + # collected until the socket it done with it. + cdef public Context context # The zmq Context object that owns this. + cdef public bint _closed # bool property for a closed socket. + cdef int _pid # the pid of the process which created me (for fork safety) + + # cpdef methods for direct-cython access: + cpdef object send(self, object data, int flags=*, copy=*, track=*) + cpdef object recv(self, int flags=*, copy=*, track=*) + diff --git a/scripts/external_libs/zmq/backend/cython/socket.py b/scripts/external_libs/zmq/backend/cython/socket.py new file mode 100644 index 00000000..faef8bee --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/socket.py @@ -0,0 +1,7 @@ +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys, pkg_resources, imp + __file__ = pkg_resources.resource_filename(__name__,'socket.so') + __loader__ = None; del __bootstrap__, __loader__ + imp.load_dynamic(__name__,__file__) +__bootstrap__() diff --git a/scripts/external_libs/zmq/backend/cython/socket.so b/scripts/external_libs/zmq/backend/cython/socket.so Binary files differnew file mode 100644 index 00000000..1c927042 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/socket.so diff --git a/scripts/external_libs/zmq/backend/cython/utils.pxd b/scripts/external_libs/zmq/backend/cython/utils.pxd new file mode 100644 index 00000000..1d7117f1 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/utils.pxd @@ -0,0 +1,29 @@ +"""Wrap zmq_utils.h""" + +# +# Copyright (c) 2010 Brian E. Granger & Min Ragan-Kelley +# +# This file is part of pyzmq. +# +# pyzmq is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# pyzmq is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- + + +cdef class Stopwatch: + cdef void *watch # The C handle for the underlying zmq object + diff --git a/scripts/external_libs/zmq/backend/cython/utils.py b/scripts/external_libs/zmq/backend/cython/utils.py new file mode 100644 index 00000000..fe928300 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/utils.py @@ -0,0 +1,7 @@ +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys, pkg_resources, imp + __file__ = pkg_resources.resource_filename(__name__,'utils.so') + __loader__ = None; del __bootstrap__, __loader__ + imp.load_dynamic(__name__,__file__) +__bootstrap__() diff --git a/scripts/external_libs/zmq/backend/cython/utils.so b/scripts/external_libs/zmq/backend/cython/utils.so Binary files differnew file mode 100644 index 00000000..b4e5b283 --- /dev/null +++ b/scripts/external_libs/zmq/backend/cython/utils.so diff --git a/scripts/external_libs/zmq/backend/select.py b/scripts/external_libs/zmq/backend/select.py new file mode 100644 index 00000000..0a2e09a2 --- /dev/null +++ b/scripts/external_libs/zmq/backend/select.py @@ -0,0 +1,39 @@ +"""Import basic exposure of libzmq C API as a backend""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +public_api = [ + 'Context', + 'Socket', + 'Frame', + 'Message', + 'Stopwatch', + 'device', + 'proxy', + 'zmq_poll', + 'strerror', + 'zmq_errno', + 'has', + 'curve_keypair', + 'constants', + 'zmq_version_info', + 'IPC_PATH_MAX_LEN', +] + +def select_backend(name): + """Select the pyzmq backend""" + try: + mod = __import__(name, fromlist=public_api) + except ImportError: + raise + except Exception as e: + import sys + from zmq.utils.sixcerpt import reraise + exc_info = sys.exc_info() + reraise(ImportError, ImportError("Importing %s failed with %s" % (name, e)), exc_info[2]) + + ns = {} + for key in public_api: + ns[key] = getattr(mod, key) + return ns diff --git a/scripts/external_libs/zmq/devices/__init__.py b/scripts/external_libs/zmq/devices/__init__.py new file mode 100644 index 00000000..23715963 --- /dev/null +++ b/scripts/external_libs/zmq/devices/__init__.py @@ -0,0 +1,16 @@ +"""0MQ Device classes for running in background threads or processes.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from zmq import device +from zmq.devices import basedevice, proxydevice, monitoredqueue, monitoredqueuedevice + +from zmq.devices.basedevice import * +from zmq.devices.proxydevice import * +from zmq.devices.monitoredqueue import * +from zmq.devices.monitoredqueuedevice import * + +__all__ = ['device'] +for submod in (basedevice, proxydevice, monitoredqueue, monitoredqueuedevice): + __all__.extend(submod.__all__) diff --git a/scripts/external_libs/zmq/devices/basedevice.py b/scripts/external_libs/zmq/devices/basedevice.py new file mode 100644 index 00000000..7ba1b7ac --- /dev/null +++ b/scripts/external_libs/zmq/devices/basedevice.py @@ -0,0 +1,229 @@ +"""Classes for running 0MQ Devices in the background.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import time +from threading import Thread +from multiprocessing import Process + +from zmq import device, QUEUE, Context, ETERM, ZMQError + + +class Device: + """A 0MQ Device to be run in the background. + + You do not pass Socket instances to this, but rather Socket types:: + + Device(device_type, in_socket_type, out_socket_type) + + For instance:: + + dev = Device(zmq.QUEUE, zmq.DEALER, zmq.ROUTER) + + Similar to zmq.device, but socket types instead of sockets themselves are + passed, and the sockets are created in the work thread, to avoid issues + with thread safety. As a result, additional bind_{in|out} and + connect_{in|out} methods and setsockopt_{in|out} allow users to specify + connections for the sockets. + + Parameters + ---------- + device_type : int + The 0MQ Device type + {in|out}_type : int + zmq socket types, to be passed later to context.socket(). e.g. + zmq.PUB, zmq.SUB, zmq.REQ. If out_type is < 0, then in_socket is used + for both in_socket and out_socket. + + Methods + ------- + bind_{in_out}(iface) + passthrough for ``{in|out}_socket.bind(iface)``, to be called in the thread + connect_{in_out}(iface) + passthrough for ``{in|out}_socket.connect(iface)``, to be called in the + thread + setsockopt_{in_out}(opt,value) + passthrough for ``{in|out}_socket.setsockopt(opt, value)``, to be called in + the thread + + Attributes + ---------- + daemon : int + sets whether the thread should be run as a daemon + Default is true, because if it is false, the thread will not + exit unless it is killed + context_factory : callable (class attribute) + Function for creating the Context. This will be Context.instance + in ThreadDevices, and Context in ProcessDevices. The only reason + it is not instance() in ProcessDevices is that there may be a stale + Context instance already initialized, and the forked environment + should *never* try to use it. + """ + + context_factory = Context.instance + """Callable that returns a context. Typically either Context.instance or Context, + depending on whether the device should share the global instance or not. + """ + + def __init__(self, device_type=QUEUE, in_type=None, out_type=None): + self.device_type = device_type + if in_type is None: + raise TypeError("in_type must be specified") + if out_type is None: + raise TypeError("out_type must be specified") + self.in_type = in_type + self.out_type = out_type + self._in_binds = [] + self._in_connects = [] + self._in_sockopts = [] + self._out_binds = [] + self._out_connects = [] + self._out_sockopts = [] + self.daemon = True + self.done = False + + def bind_in(self, addr): + """Enqueue ZMQ address for binding on in_socket. + + See zmq.Socket.bind for details. + """ + self._in_binds.append(addr) + + def connect_in(self, addr): + """Enqueue ZMQ address for connecting on in_socket. + + See zmq.Socket.connect for details. + """ + self._in_connects.append(addr) + + def setsockopt_in(self, opt, value): + """Enqueue setsockopt(opt, value) for in_socket + + See zmq.Socket.setsockopt for details. + """ + self._in_sockopts.append((opt, value)) + + def bind_out(self, addr): + """Enqueue ZMQ address for binding on out_socket. + + See zmq.Socket.bind for details. + """ + self._out_binds.append(addr) + + def connect_out(self, addr): + """Enqueue ZMQ address for connecting on out_socket. + + See zmq.Socket.connect for details. + """ + self._out_connects.append(addr) + + def setsockopt_out(self, opt, value): + """Enqueue setsockopt(opt, value) for out_socket + + See zmq.Socket.setsockopt for details. + """ + self._out_sockopts.append((opt, value)) + + def _setup_sockets(self): + ctx = self.context_factory() + + self._context = ctx + + # create the sockets + ins = ctx.socket(self.in_type) + if self.out_type < 0: + outs = ins + else: + outs = ctx.socket(self.out_type) + + # set sockopts (must be done first, in case of zmq.IDENTITY) + for opt,value in self._in_sockopts: + ins.setsockopt(opt, value) + for opt,value in self._out_sockopts: + outs.setsockopt(opt, value) + + for iface in self._in_binds: + ins.bind(iface) + for iface in self._out_binds: + outs.bind(iface) + + for iface in self._in_connects: + ins.connect(iface) + for iface in self._out_connects: + outs.connect(iface) + + return ins,outs + + def run_device(self): + """The runner method. + + Do not call me directly, instead call ``self.start()``, just like a Thread. + """ + ins,outs = self._setup_sockets() + device(self.device_type, ins, outs) + + def run(self): + """wrap run_device in try/catch ETERM""" + try: + self.run_device() + except ZMQError as e: + if e.errno == ETERM: + # silence TERM errors, because this should be a clean shutdown + pass + else: + raise + finally: + self.done = True + + def start(self): + """Start the device. Override me in subclass for other launchers.""" + return self.run() + + def join(self,timeout=None): + """wait for me to finish, like Thread.join. + + Reimplemented appropriately by subclasses.""" + tic = time.time() + toc = tic + while not self.done and not (timeout is not None and toc-tic > timeout): + time.sleep(.001) + toc = time.time() + + +class BackgroundDevice(Device): + """Base class for launching Devices in background processes and threads.""" + + launcher=None + _launch_class=None + + def start(self): + self.launcher = self._launch_class(target=self.run) + self.launcher.daemon = self.daemon + return self.launcher.start() + + def join(self, timeout=None): + return self.launcher.join(timeout=timeout) + + +class ThreadDevice(BackgroundDevice): + """A Device that will be run in a background Thread. + + See Device for details. + """ + _launch_class=Thread + +class ProcessDevice(BackgroundDevice): + """A Device that will be run in a background Process. + + See Device for details. + """ + _launch_class=Process + context_factory = Context + """Callable that returns a context. Typically either Context.instance or Context, + depending on whether the device should share the global instance or not. + """ + + +__all__ = ['Device', 'ThreadDevice', 'ProcessDevice'] diff --git a/scripts/external_libs/zmq/devices/monitoredqueue.pxd b/scripts/external_libs/zmq/devices/monitoredqueue.pxd new file mode 100644 index 00000000..1e26ed86 --- /dev/null +++ b/scripts/external_libs/zmq/devices/monitoredqueue.pxd @@ -0,0 +1,177 @@ +"""MonitoredQueue class declarations. + +Authors +------- +* MinRK +* Brian Granger +""" + +# +# Copyright (c) 2010 Min Ragan-Kelley, Brian Granger +# +# This file is part of pyzmq, but is derived and adapted from zmq_queue.cpp +# originally from libzmq-2.1.6, used under LGPLv3 +# +# pyzmq is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# pyzmq is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from libzmq cimport * + +#----------------------------------------------------------------------------- +# MonitoredQueue C functions +#----------------------------------------------------------------------------- + +cdef inline int _relay(void *insocket_, void *outsocket_, void *sidesocket_, + zmq_msg_t msg, zmq_msg_t side_msg, zmq_msg_t id_msg, + bint swap_ids) nogil: + cdef int rc + cdef int64_t flag_2 + cdef int flag_3 + cdef int flags + cdef bint more + cdef size_t flagsz + cdef void * flag_ptr + + if ZMQ_VERSION_MAJOR < 3: + flagsz = sizeof (int64_t) + flag_ptr = &flag_2 + else: + flagsz = sizeof (int) + flag_ptr = &flag_3 + + if swap_ids:# both router, must send second identity first + # recv two ids into msg, id_msg + rc = zmq_msg_recv(&msg, insocket_, 0) + if rc < 0: return rc + + rc = zmq_msg_recv(&id_msg, insocket_, 0) + if rc < 0: return rc + + # send second id (id_msg) first + #!!!! always send a copy before the original !!!! + rc = zmq_msg_copy(&side_msg, &id_msg) + if rc < 0: return rc + rc = zmq_msg_send(&side_msg, outsocket_, ZMQ_SNDMORE) + if rc < 0: return rc + rc = zmq_msg_send(&id_msg, sidesocket_, ZMQ_SNDMORE) + if rc < 0: return rc + # send first id (msg) second + rc = zmq_msg_copy(&side_msg, &msg) + if rc < 0: return rc + rc = zmq_msg_send(&side_msg, outsocket_, ZMQ_SNDMORE) + if rc < 0: return rc + rc = zmq_msg_send(&msg, sidesocket_, ZMQ_SNDMORE) + if rc < 0: return rc + while (True): + rc = zmq_msg_recv(&msg, insocket_, 0) + if rc < 0: return rc + # assert (rc == 0) + rc = zmq_getsockopt (insocket_, ZMQ_RCVMORE, flag_ptr, &flagsz) + if rc < 0: return rc + flags = 0 + if ZMQ_VERSION_MAJOR < 3: + if flag_2: + flags |= ZMQ_SNDMORE + else: + if flag_3: + flags |= ZMQ_SNDMORE + # LABEL has been removed: + # rc = zmq_getsockopt (insocket_, ZMQ_RCVLABEL, flag_ptr, &flagsz) + # if flag_3: + # flags |= ZMQ_SNDLABEL + # assert (rc == 0) + + rc = zmq_msg_copy(&side_msg, &msg) + if rc < 0: return rc + if flags: + rc = zmq_msg_send(&side_msg, outsocket_, flags) + if rc < 0: return rc + # only SNDMORE for side-socket + rc = zmq_msg_send(&msg, sidesocket_, ZMQ_SNDMORE) + if rc < 0: return rc + else: + rc = zmq_msg_send(&side_msg, outsocket_, 0) + if rc < 0: return rc + rc = zmq_msg_send(&msg, sidesocket_, 0) + if rc < 0: return rc + break + return rc + +# the MonitoredQueue C function, adapted from zmq::queue.cpp : +cdef inline int c_monitored_queue (void *insocket_, void *outsocket_, + void *sidesocket_, zmq_msg_t *in_msg_ptr, + zmq_msg_t *out_msg_ptr, int swap_ids) nogil: + """The actual C function for a monitored queue device. + + See ``monitored_queue()`` for details. + """ + + cdef zmq_msg_t msg + cdef int rc = zmq_msg_init (&msg) + cdef zmq_msg_t id_msg + rc = zmq_msg_init (&id_msg) + if rc < 0: return rc + cdef zmq_msg_t side_msg + rc = zmq_msg_init (&side_msg) + if rc < 0: return rc + + cdef zmq_pollitem_t items [2] + items [0].socket = insocket_ + items [0].fd = 0 + items [0].events = ZMQ_POLLIN + items [0].revents = 0 + items [1].socket = outsocket_ + items [1].fd = 0 + items [1].events = ZMQ_POLLIN + items [1].revents = 0 + # I don't think sidesocket should be polled? + # items [2].socket = sidesocket_ + # items [2].fd = 0 + # items [2].events = ZMQ_POLLIN + # items [2].revents = 0 + + while (True): + + # // Wait while there are either requests or replies to process. + rc = zmq_poll (&items [0], 2, -1) + if rc < 0: return rc + # // The algorithm below asumes ratio of request and replies processed + # // under full load to be 1:1. Although processing requests replies + # // first is tempting it is suspectible to DoS attacks (overloading + # // the system with unsolicited replies). + # + # // Process a request. + if (items [0].revents & ZMQ_POLLIN): + # send in_prefix to side socket + rc = zmq_msg_copy(&side_msg, in_msg_ptr) + if rc < 0: return rc + rc = zmq_msg_send(&side_msg, sidesocket_, ZMQ_SNDMORE) + if rc < 0: return rc + # relay the rest of the message + rc = _relay(insocket_, outsocket_, sidesocket_, msg, side_msg, id_msg, swap_ids) + if rc < 0: return rc + if (items [1].revents & ZMQ_POLLIN): + # send out_prefix to side socket + rc = zmq_msg_copy(&side_msg, out_msg_ptr) + if rc < 0: return rc + rc = zmq_msg_send(&side_msg, sidesocket_, ZMQ_SNDMORE) + if rc < 0: return rc + # relay the rest of the message + rc = _relay(outsocket_, insocket_, sidesocket_, msg, side_msg, id_msg, swap_ids) + if rc < 0: return rc + return rc diff --git a/scripts/external_libs/zmq/devices/monitoredqueue.py b/scripts/external_libs/zmq/devices/monitoredqueue.py new file mode 100644 index 00000000..6d714e51 --- /dev/null +++ b/scripts/external_libs/zmq/devices/monitoredqueue.py @@ -0,0 +1,7 @@ +def __bootstrap__(): + global __bootstrap__, __loader__, __file__ + import sys, pkg_resources, imp + __file__ = pkg_resources.resource_filename(__name__,'monitoredqueue.so') + __loader__ = None; del __bootstrap__, __loader__ + imp.load_dynamic(__name__,__file__) +__bootstrap__() diff --git a/scripts/external_libs/zmq/devices/monitoredqueue.so b/scripts/external_libs/zmq/devices/monitoredqueue.so Binary files differnew file mode 100644 index 00000000..edca8a4b --- /dev/null +++ b/scripts/external_libs/zmq/devices/monitoredqueue.so diff --git a/scripts/external_libs/zmq/devices/monitoredqueuedevice.py b/scripts/external_libs/zmq/devices/monitoredqueuedevice.py new file mode 100644 index 00000000..9723f866 --- /dev/null +++ b/scripts/external_libs/zmq/devices/monitoredqueuedevice.py @@ -0,0 +1,66 @@ +"""MonitoredQueue classes and functions.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +from zmq import ZMQError, PUB +from zmq.devices.proxydevice import ProxyBase, Proxy, ThreadProxy, ProcessProxy +from zmq.devices.monitoredqueue import monitored_queue + + +class MonitoredQueueBase(ProxyBase): + """Base class for overriding methods.""" + + _in_prefix = b'' + _out_prefix = b'' + + def __init__(self, in_type, out_type, mon_type=PUB, in_prefix=b'in', out_prefix=b'out'): + + ProxyBase.__init__(self, in_type=in_type, out_type=out_type, mon_type=mon_type) + + self._in_prefix = in_prefix + self._out_prefix = out_prefix + + def run_device(self): + ins,outs,mons = self._setup_sockets() + monitored_queue(ins, outs, mons, self._in_prefix, self._out_prefix) + + +class MonitoredQueue(MonitoredQueueBase, Proxy): + """Class for running monitored_queue in the background. + + See zmq.devices.Device for most of the spec. MonitoredQueue differs from Proxy, + only in that it adds a ``prefix`` to messages sent on the monitor socket, + with a different prefix for each direction. + + MQ also supports ROUTER on both sides, which zmq.proxy does not. + + If a message arrives on `in_sock`, it will be prefixed with `in_prefix` on the monitor socket. + If it arrives on out_sock, it will be prefixed with `out_prefix`. + + A PUB socket is the most logical choice for the mon_socket, but it is not required. + """ + pass + + +class ThreadMonitoredQueue(MonitoredQueueBase, ThreadProxy): + """Run zmq.monitored_queue in a background thread. + + See MonitoredQueue and Proxy for details. + """ + pass + + +class ProcessMonitoredQueue(MonitoredQueueBase, ProcessProxy): + """Run zmq.monitored_queue in a background thread. + + See MonitoredQueue and Proxy for details. + """ + + +__all__ = [ + 'MonitoredQueue', + 'ThreadMonitoredQueue', + 'ProcessMonitoredQueue' +] diff --git a/scripts/external_libs/zmq/devices/proxydevice.py b/scripts/external_libs/zmq/devices/proxydevice.py new file mode 100644 index 00000000..68be3f15 --- /dev/null +++ b/scripts/external_libs/zmq/devices/proxydevice.py @@ -0,0 +1,90 @@ +"""Proxy classes and functions.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import zmq +from zmq.devices.basedevice import Device, ThreadDevice, ProcessDevice + + +class ProxyBase(object): + """Base class for overriding methods.""" + + def __init__(self, in_type, out_type, mon_type=zmq.PUB): + + Device.__init__(self, in_type=in_type, out_type=out_type) + self.mon_type = mon_type + self._mon_binds = [] + self._mon_connects = [] + self._mon_sockopts = [] + + def bind_mon(self, addr): + """Enqueue ZMQ address for binding on mon_socket. + + See zmq.Socket.bind for details. + """ + self._mon_binds.append(addr) + + def connect_mon(self, addr): + """Enqueue ZMQ address for connecting on mon_socket. + + See zmq.Socket.bind for details. + """ + self._mon_connects.append(addr) + + def setsockopt_mon(self, opt, value): + """Enqueue setsockopt(opt, value) for mon_socket + + See zmq.Socket.setsockopt for details. + """ + self._mon_sockopts.append((opt, value)) + + def _setup_sockets(self): + ins,outs = Device._setup_sockets(self) + ctx = self._context + mons = ctx.socket(self.mon_type) + + # set sockopts (must be done first, in case of zmq.IDENTITY) + for opt,value in self._mon_sockopts: + mons.setsockopt(opt, value) + + for iface in self._mon_binds: + mons.bind(iface) + + for iface in self._mon_connects: + mons.connect(iface) + + return ins,outs,mons + + def run_device(self): + ins,outs,mons = self._setup_sockets() + zmq.proxy(ins, outs, mons) + +class Proxy(ProxyBase, Device): + """Threadsafe Proxy object. + + See zmq.devices.Device for most of the spec. This subclass adds a + <method>_mon version of each <method>_{in|out} method, for configuring the + monitor socket. + + A Proxy is a 3-socket ZMQ Device that functions just like a + QUEUE, except each message is also sent out on the monitor socket. + + A PUB socket is the most logical choice for the mon_socket, but it is not required. + """ + pass + +class ThreadProxy(ProxyBase, ThreadDevice): + """Proxy in a Thread. See Proxy for more.""" + pass + +class ProcessProxy(ProxyBase, ProcessDevice): + """Proxy in a Process. See Proxy for more.""" + pass + + +__all__ = [ + 'Proxy', + 'ThreadProxy', + 'ProcessProxy', +] diff --git a/scripts/external_libs/zmq/error.py b/scripts/external_libs/zmq/error.py new file mode 100644 index 00000000..48cdaafa --- /dev/null +++ b/scripts/external_libs/zmq/error.py @@ -0,0 +1,164 @@ +"""0MQ Error classes and functions.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +class ZMQBaseError(Exception): + """Base exception class for 0MQ errors in Python.""" + pass + +class ZMQError(ZMQBaseError): + """Wrap an errno style error. + + Parameters + ---------- + errno : int + The ZMQ errno or None. If None, then ``zmq_errno()`` is called and + used. + msg : string + Description of the error or None. + """ + errno = None + + def __init__(self, errno=None, msg=None): + """Wrap an errno style error. + + Parameters + ---------- + errno : int + The ZMQ errno or None. If None, then ``zmq_errno()`` is called and + used. + msg : string + Description of the error or None. + """ + from zmq.backend import strerror, zmq_errno + if errno is None: + errno = zmq_errno() + if isinstance(errno, int): + self.errno = errno + if msg is None: + self.strerror = strerror(errno) + else: + self.strerror = msg + else: + if msg is None: + self.strerror = str(errno) + else: + self.strerror = msg + # flush signals, because there could be a SIGINT + # waiting to pounce, resulting in uncaught exceptions. + # Doing this here means getting SIGINT during a blocking + # libzmq call will raise a *catchable* KeyboardInterrupt + # PyErr_CheckSignals() + + def __str__(self): + return self.strerror + + def __repr__(self): + return "ZMQError('%s')"%self.strerror + + +class ZMQBindError(ZMQBaseError): + """An error for ``Socket.bind_to_random_port()``. + + See Also + -------- + .Socket.bind_to_random_port + """ + pass + + +class NotDone(ZMQBaseError): + """Raised when timeout is reached while waiting for 0MQ to finish with a Message + + See Also + -------- + .MessageTracker.wait : object for tracking when ZeroMQ is done + """ + pass + + +class ContextTerminated(ZMQError): + """Wrapper for zmq.ETERM + + .. versionadded:: 13.0 + """ + pass + + +class Again(ZMQError): + """Wrapper for zmq.EAGAIN + + .. versionadded:: 13.0 + """ + pass + + +def _check_rc(rc, errno=None): + """internal utility for checking zmq return condition + + and raising the appropriate Exception class + """ + if rc < 0: + from zmq.backend import zmq_errno + if errno is None: + errno = zmq_errno() + from zmq import EAGAIN, ETERM + if errno == EAGAIN: + raise Again(errno) + elif errno == ETERM: + raise ContextTerminated(errno) + else: + raise ZMQError(errno) + +_zmq_version_info = None +_zmq_version = None + +class ZMQVersionError(NotImplementedError): + """Raised when a feature is not provided by the linked version of libzmq. + + .. versionadded:: 14.2 + """ + min_version = None + def __init__(self, min_version, msg='Feature'): + global _zmq_version + if _zmq_version is None: + from zmq import zmq_version + _zmq_version = zmq_version() + self.msg = msg + self.min_version = min_version + self.version = _zmq_version + + def __repr__(self): + return "ZMQVersionError('%s')" % str(self) + + def __str__(self): + return "%s requires libzmq >= %s, have %s" % (self.msg, self.min_version, self.version) + + +def _check_version(min_version_info, msg='Feature'): + """Check for libzmq + + raises ZMQVersionError if current zmq version is not at least min_version + + min_version_info is a tuple of integers, and will be compared against zmq.zmq_version_info(). + """ + global _zmq_version_info + if _zmq_version_info is None: + from zmq import zmq_version_info + _zmq_version_info = zmq_version_info() + if _zmq_version_info < min_version_info: + min_version = '.'.join(str(v) for v in min_version_info) + raise ZMQVersionError(min_version, msg) + + +__all__ = [ + 'ZMQBaseError', + 'ZMQBindError', + 'ZMQError', + 'NotDone', + 'ContextTerminated', + 'Again', + 'ZMQVersionError', +] diff --git a/scripts/external_libs/zmq/eventloop/__init__.py b/scripts/external_libs/zmq/eventloop/__init__.py new file mode 100644 index 00000000..568e8e8d --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/__init__.py @@ -0,0 +1,5 @@ +"""A Tornado based event loop for PyZMQ.""" + +from zmq.eventloop.ioloop import IOLoop + +__all__ = ['IOLoop']
\ No newline at end of file diff --git a/scripts/external_libs/zmq/eventloop/ioloop.py b/scripts/external_libs/zmq/eventloop/ioloop.py new file mode 100644 index 00000000..35f4c418 --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/ioloop.py @@ -0,0 +1,193 @@ +# coding: utf-8 +"""tornado IOLoop API with zmq compatibility + +If you have tornado ≥ 3.0, this is a subclass of tornado's IOLoop, +otherwise we ship a minimal subset of tornado in zmq.eventloop.minitornado. + +The minimal shipped version of tornado's IOLoop does not include +support for concurrent futures - this will only be available if you +have tornado ≥ 3.0. +""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from __future__ import absolute_import, division, with_statement + +import os +import time +import warnings + +from zmq import ( + Poller, + POLLIN, POLLOUT, POLLERR, + ZMQError, ETERM, +) + +try: + import tornado + tornado_version = tornado.version_info +except (ImportError, AttributeError): + tornado_version = () + +try: + # tornado ≥ 3 + from tornado.ioloop import PollIOLoop, PeriodicCallback + from tornado.log import gen_log +except ImportError: + from .minitornado.ioloop import PollIOLoop, PeriodicCallback + from .minitornado.log import gen_log + + +class DelayedCallback(PeriodicCallback): + """Schedules the given callback to be called once. + + The callback is called once, after callback_time milliseconds. + + `start` must be called after the DelayedCallback is created. + + The timeout is calculated from when `start` is called. + """ + def __init__(self, callback, callback_time, io_loop=None): + # PeriodicCallback require callback_time to be positive + warnings.warn("""DelayedCallback is deprecated. + Use loop.add_timeout instead.""", DeprecationWarning) + callback_time = max(callback_time, 1e-3) + super(DelayedCallback, self).__init__(callback, callback_time, io_loop) + + def start(self): + """Starts the timer.""" + self._running = True + self._firstrun = True + self._next_timeout = time.time() + self.callback_time / 1000.0 + self.io_loop.add_timeout(self._next_timeout, self._run) + + def _run(self): + if not self._running: return + self._running = False + try: + self.callback() + except Exception: + gen_log.error("Error in delayed callback", exc_info=True) + + +class ZMQPoller(object): + """A poller that can be used in the tornado IOLoop. + + This simply wraps a regular zmq.Poller, scaling the timeout + by 1000, so that it is in seconds rather than milliseconds. + """ + + def __init__(self): + self._poller = Poller() + + @staticmethod + def _map_events(events): + """translate IOLoop.READ/WRITE/ERROR event masks into zmq.POLLIN/OUT/ERR""" + z_events = 0 + if events & IOLoop.READ: + z_events |= POLLIN + if events & IOLoop.WRITE: + z_events |= POLLOUT + if events & IOLoop.ERROR: + z_events |= POLLERR + return z_events + + @staticmethod + def _remap_events(z_events): + """translate zmq.POLLIN/OUT/ERR event masks into IOLoop.READ/WRITE/ERROR""" + events = 0 + if z_events & POLLIN: + events |= IOLoop.READ + if z_events & POLLOUT: + events |= IOLoop.WRITE + if z_events & POLLERR: + events |= IOLoop.ERROR + return events + + def register(self, fd, events): + return self._poller.register(fd, self._map_events(events)) + + def modify(self, fd, events): + return self._poller.modify(fd, self._map_events(events)) + + def unregister(self, fd): + return self._poller.unregister(fd) + + def poll(self, timeout): + """poll in seconds rather than milliseconds. + + Event masks will be IOLoop.READ/WRITE/ERROR + """ + z_events = self._poller.poll(1000*timeout) + return [ (fd,self._remap_events(evt)) for (fd,evt) in z_events ] + + def close(self): + pass + + +class ZMQIOLoop(PollIOLoop): + """ZMQ subclass of tornado's IOLoop""" + def initialize(self, impl=None, **kwargs): + impl = ZMQPoller() if impl is None else impl + super(ZMQIOLoop, self).initialize(impl=impl, **kwargs) + + @staticmethod + def instance(): + """Returns a global `IOLoop` instance. + + Most applications have a single, global `IOLoop` running on the + main thread. Use this method to get this instance from + another thread. To get the current thread's `IOLoop`, use `current()`. + """ + # install ZMQIOLoop as the active IOLoop implementation + # when using tornado 3 + if tornado_version >= (3,): + PollIOLoop.configure(ZMQIOLoop) + return PollIOLoop.instance() + + def start(self): + try: + super(ZMQIOLoop, self).start() + except ZMQError as e: + if e.errno == ETERM: + # quietly return on ETERM + pass + else: + raise e + + +if tornado_version >= (3,0) and tornado_version < (3,1): + def backport_close(self, all_fds=False): + """backport IOLoop.close to 3.0 from 3.1 (supports fd.close() method)""" + from zmq.eventloop.minitornado.ioloop import PollIOLoop as mini_loop + return mini_loop.close.__get__(self)(all_fds) + ZMQIOLoop.close = backport_close + + +# public API name +IOLoop = ZMQIOLoop + + +def install(): + """set the tornado IOLoop instance with the pyzmq IOLoop. + + After calling this function, tornado's IOLoop.instance() and pyzmq's + IOLoop.instance() will return the same object. + + An assertion error will be raised if tornado's IOLoop has been initialized + prior to calling this function. + """ + from tornado import ioloop + # check if tornado's IOLoop is already initialized to something other + # than the pyzmq IOLoop instance: + assert (not ioloop.IOLoop.initialized()) or \ + ioloop.IOLoop.instance() is IOLoop.instance(), "tornado IOLoop already initialized" + + if tornado_version >= (3,): + # tornado 3 has an official API for registering new defaults, yay! + ioloop.IOLoop.configure(ZMQIOLoop) + else: + # we have to set the global instance explicitly + ioloop.IOLoop._instance = IOLoop.instance() + diff --git a/scripts/external_libs/zmq/eventloop/minitornado/__init__.py b/scripts/external_libs/zmq/eventloop/minitornado/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/__init__.py diff --git a/scripts/external_libs/zmq/eventloop/minitornado/concurrent.py b/scripts/external_libs/zmq/eventloop/minitornado/concurrent.py new file mode 100644 index 00000000..519b23d5 --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/concurrent.py @@ -0,0 +1,11 @@ +"""pyzmq does not ship tornado's futures, +this just raises informative NotImplementedErrors to avoid having to change too much code. +""" + +class NotImplementedFuture(object): + def __init__(self, *args, **kwargs): + raise NotImplementedError("pyzmq does not ship tornado's Futures, " + "install tornado >= 3.0 for future support." + ) + +Future = TracebackFuture = NotImplementedFuture diff --git a/scripts/external_libs/zmq/eventloop/minitornado/ioloop.py b/scripts/external_libs/zmq/eventloop/minitornado/ioloop.py new file mode 100644 index 00000000..710a3ecb --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/ioloop.py @@ -0,0 +1,829 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""An I/O event loop for non-blocking sockets. + +Typical applications will use a single `IOLoop` object, in the +`IOLoop.instance` singleton. The `IOLoop.start` method should usually +be called at the end of the ``main()`` function. Atypical applications may +use more than one `IOLoop`, such as one `IOLoop` per thread, or per `unittest` +case. + +In addition to I/O events, the `IOLoop` can also schedule time-based events. +`IOLoop.add_timeout` is a non-blocking alternative to `time.sleep`. +""" + +from __future__ import absolute_import, division, print_function, with_statement + +import datetime +import errno +import functools +import heapq +import logging +import numbers +import os +import select +import sys +import threading +import time +import traceback + +from .concurrent import Future, TracebackFuture +from .log import app_log, gen_log +from . import stack_context +from .util import Configurable + +try: + import signal +except ImportError: + signal = None + +try: + import thread # py2 +except ImportError: + import _thread as thread # py3 + +from .platform.auto import set_close_exec, Waker + + +class TimeoutError(Exception): + pass + + +class IOLoop(Configurable): + """A level-triggered I/O loop. + + We use ``epoll`` (Linux) or ``kqueue`` (BSD and Mac OS X) if they + are available, or else we fall back on select(). If you are + implementing a system that needs to handle thousands of + simultaneous connections, you should use a system that supports + either ``epoll`` or ``kqueue``. + + Example usage for a simple TCP server:: + + import errno + import functools + import ioloop + import socket + + def connection_ready(sock, fd, events): + while True: + try: + connection, address = sock.accept() + except socket.error, e: + if e.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN): + raise + return + connection.setblocking(0) + handle_connection(connection, address) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setblocking(0) + sock.bind(("", port)) + sock.listen(128) + + io_loop = ioloop.IOLoop.instance() + callback = functools.partial(connection_ready, sock) + io_loop.add_handler(sock.fileno(), callback, io_loop.READ) + io_loop.start() + + """ + # Constants from the epoll module + _EPOLLIN = 0x001 + _EPOLLPRI = 0x002 + _EPOLLOUT = 0x004 + _EPOLLERR = 0x008 + _EPOLLHUP = 0x010 + _EPOLLRDHUP = 0x2000 + _EPOLLONESHOT = (1 << 30) + _EPOLLET = (1 << 31) + + # Our events map exactly to the epoll events + NONE = 0 + READ = _EPOLLIN + WRITE = _EPOLLOUT + ERROR = _EPOLLERR | _EPOLLHUP + + # Global lock for creating global IOLoop instance + _instance_lock = threading.Lock() + + _current = threading.local() + + @staticmethod + def instance(): + """Returns a global `IOLoop` instance. + + Most applications have a single, global `IOLoop` running on the + main thread. Use this method to get this instance from + another thread. To get the current thread's `IOLoop`, use `current()`. + """ + if not hasattr(IOLoop, "_instance"): + with IOLoop._instance_lock: + if not hasattr(IOLoop, "_instance"): + # New instance after double check + IOLoop._instance = IOLoop() + return IOLoop._instance + + @staticmethod + def initialized(): + """Returns true if the singleton instance has been created.""" + return hasattr(IOLoop, "_instance") + + def install(self): + """Installs this `IOLoop` object as the singleton instance. + + This is normally not necessary as `instance()` will create + an `IOLoop` on demand, but you may want to call `install` to use + a custom subclass of `IOLoop`. + """ + assert not IOLoop.initialized() + IOLoop._instance = self + + @staticmethod + def current(): + """Returns the current thread's `IOLoop`. + + If an `IOLoop` is currently running or has been marked as current + by `make_current`, returns that instance. Otherwise returns + `IOLoop.instance()`, i.e. the main thread's `IOLoop`. + + A common pattern for classes that depend on ``IOLoops`` is to use + a default argument to enable programs with multiple ``IOLoops`` + but not require the argument for simpler applications:: + + class MyClass(object): + def __init__(self, io_loop=None): + self.io_loop = io_loop or IOLoop.current() + + In general you should use `IOLoop.current` as the default when + constructing an asynchronous object, and use `IOLoop.instance` + when you mean to communicate to the main thread from a different + one. + """ + current = getattr(IOLoop._current, "instance", None) + if current is None: + return IOLoop.instance() + return current + + def make_current(self): + """Makes this the `IOLoop` for the current thread. + + An `IOLoop` automatically becomes current for its thread + when it is started, but it is sometimes useful to call + `make_current` explictly before starting the `IOLoop`, + so that code run at startup time can find the right + instance. + """ + IOLoop._current.instance = self + + @staticmethod + def clear_current(): + IOLoop._current.instance = None + + @classmethod + def configurable_base(cls): + return IOLoop + + @classmethod + def configurable_default(cls): + # this is the only patch to IOLoop: + from zmq.eventloop.ioloop import ZMQIOLoop + return ZMQIOLoop + # the remainder of this method is unused, + # but left for preservation reasons + if hasattr(select, "epoll"): + from tornado.platform.epoll import EPollIOLoop + return EPollIOLoop + if hasattr(select, "kqueue"): + # Python 2.6+ on BSD or Mac + from tornado.platform.kqueue import KQueueIOLoop + return KQueueIOLoop + from tornado.platform.select import SelectIOLoop + return SelectIOLoop + + def initialize(self): + pass + + def close(self, all_fds=False): + """Closes the `IOLoop`, freeing any resources used. + + If ``all_fds`` is true, all file descriptors registered on the + IOLoop will be closed (not just the ones created by the + `IOLoop` itself). + + Many applications will only use a single `IOLoop` that runs for the + entire lifetime of the process. In that case closing the `IOLoop` + is not necessary since everything will be cleaned up when the + process exits. `IOLoop.close` is provided mainly for scenarios + such as unit tests, which create and destroy a large number of + ``IOLoops``. + + An `IOLoop` must be completely stopped before it can be closed. This + means that `IOLoop.stop()` must be called *and* `IOLoop.start()` must + be allowed to return before attempting to call `IOLoop.close()`. + Therefore the call to `close` will usually appear just after + the call to `start` rather than near the call to `stop`. + + .. versionchanged:: 3.1 + If the `IOLoop` implementation supports non-integer objects + for "file descriptors", those objects will have their + ``close`` method when ``all_fds`` is true. + """ + raise NotImplementedError() + + def add_handler(self, fd, handler, events): + """Registers the given handler to receive the given events for fd. + + The ``events`` argument is a bitwise or of the constants + ``IOLoop.READ``, ``IOLoop.WRITE``, and ``IOLoop.ERROR``. + + When an event occurs, ``handler(fd, events)`` will be run. + """ + raise NotImplementedError() + + def update_handler(self, fd, events): + """Changes the events we listen for fd.""" + raise NotImplementedError() + + def remove_handler(self, fd): + """Stop listening for events on fd.""" + raise NotImplementedError() + + def set_blocking_signal_threshold(self, seconds, action): + """Sends a signal if the `IOLoop` is blocked for more than + ``s`` seconds. + + Pass ``seconds=None`` to disable. Requires Python 2.6 on a unixy + platform. + + The action parameter is a Python signal handler. Read the + documentation for the `signal` module for more information. + If ``action`` is None, the process will be killed if it is + blocked for too long. + """ + raise NotImplementedError() + + def set_blocking_log_threshold(self, seconds): + """Logs a stack trace if the `IOLoop` is blocked for more than + ``s`` seconds. + + Equivalent to ``set_blocking_signal_threshold(seconds, + self.log_stack)`` + """ + self.set_blocking_signal_threshold(seconds, self.log_stack) + + def log_stack(self, signal, frame): + """Signal handler to log the stack trace of the current thread. + + For use with `set_blocking_signal_threshold`. + """ + gen_log.warning('IOLoop blocked for %f seconds in\n%s', + self._blocking_signal_threshold, + ''.join(traceback.format_stack(frame))) + + def start(self): + """Starts the I/O loop. + + The loop will run until one of the callbacks calls `stop()`, which + will make the loop stop after the current event iteration completes. + """ + raise NotImplementedError() + + def stop(self): + """Stop the I/O loop. + + If the event loop is not currently running, the next call to `start()` + will return immediately. + + To use asynchronous methods from otherwise-synchronous code (such as + unit tests), you can start and stop the event loop like this:: + + ioloop = IOLoop() + async_method(ioloop=ioloop, callback=ioloop.stop) + ioloop.start() + + ``ioloop.start()`` will return after ``async_method`` has run + its callback, whether that callback was invoked before or + after ``ioloop.start``. + + Note that even after `stop` has been called, the `IOLoop` is not + completely stopped until `IOLoop.start` has also returned. + Some work that was scheduled before the call to `stop` may still + be run before the `IOLoop` shuts down. + """ + raise NotImplementedError() + + def run_sync(self, func, timeout=None): + """Starts the `IOLoop`, runs the given function, and stops the loop. + + If the function returns a `.Future`, the `IOLoop` will run + until the future is resolved. If it raises an exception, the + `IOLoop` will stop and the exception will be re-raised to the + caller. + + The keyword-only argument ``timeout`` may be used to set + a maximum duration for the function. If the timeout expires, + a `TimeoutError` is raised. + + This method is useful in conjunction with `tornado.gen.coroutine` + to allow asynchronous calls in a ``main()`` function:: + + @gen.coroutine + def main(): + # do stuff... + + if __name__ == '__main__': + IOLoop.instance().run_sync(main) + """ + future_cell = [None] + + def run(): + try: + result = func() + except Exception: + future_cell[0] = TracebackFuture() + future_cell[0].set_exc_info(sys.exc_info()) + else: + if isinstance(result, Future): + future_cell[0] = result + else: + future_cell[0] = Future() + future_cell[0].set_result(result) + self.add_future(future_cell[0], lambda future: self.stop()) + self.add_callback(run) + if timeout is not None: + timeout_handle = self.add_timeout(self.time() + timeout, self.stop) + self.start() + if timeout is not None: + self.remove_timeout(timeout_handle) + if not future_cell[0].done(): + raise TimeoutError('Operation timed out after %s seconds' % timeout) + return future_cell[0].result() + + def time(self): + """Returns the current time according to the `IOLoop`'s clock. + + The return value is a floating-point number relative to an + unspecified time in the past. + + By default, the `IOLoop`'s time function is `time.time`. However, + it may be configured to use e.g. `time.monotonic` instead. + Calls to `add_timeout` that pass a number instead of a + `datetime.timedelta` should use this function to compute the + appropriate time, so they can work no matter what time function + is chosen. + """ + return time.time() + + def add_timeout(self, deadline, callback): + """Runs the ``callback`` at the time ``deadline`` from the I/O loop. + + Returns an opaque handle that may be passed to + `remove_timeout` to cancel. + + ``deadline`` may be a number denoting a time (on the same + scale as `IOLoop.time`, normally `time.time`), or a + `datetime.timedelta` object for a deadline relative to the + current time. + + Note that it is not safe to call `add_timeout` from other threads. + Instead, you must use `add_callback` to transfer control to the + `IOLoop`'s thread, and then call `add_timeout` from there. + """ + raise NotImplementedError() + + def remove_timeout(self, timeout): + """Cancels a pending timeout. + + The argument is a handle as returned by `add_timeout`. It is + safe to call `remove_timeout` even if the callback has already + been run. + """ + raise NotImplementedError() + + def add_callback(self, callback, *args, **kwargs): + """Calls the given callback on the next I/O loop iteration. + + It is safe to call this method from any thread at any time, + except from a signal handler. Note that this is the **only** + method in `IOLoop` that makes this thread-safety guarantee; all + other interaction with the `IOLoop` must be done from that + `IOLoop`'s thread. `add_callback()` may be used to transfer + control from other threads to the `IOLoop`'s thread. + + To add a callback from a signal handler, see + `add_callback_from_signal`. + """ + raise NotImplementedError() + + def add_callback_from_signal(self, callback, *args, **kwargs): + """Calls the given callback on the next I/O loop iteration. + + Safe for use from a Python signal handler; should not be used + otherwise. + + Callbacks added with this method will be run without any + `.stack_context`, to avoid picking up the context of the function + that was interrupted by the signal. + """ + raise NotImplementedError() + + def add_future(self, future, callback): + """Schedules a callback on the ``IOLoop`` when the given + `.Future` is finished. + + The callback is invoked with one argument, the + `.Future`. + """ + assert isinstance(future, Future) + callback = stack_context.wrap(callback) + future.add_done_callback( + lambda future: self.add_callback(callback, future)) + + def _run_callback(self, callback): + """Runs a callback with error handling. + + For use in subclasses. + """ + try: + callback() + except Exception: + self.handle_callback_exception(callback) + + def handle_callback_exception(self, callback): + """This method is called whenever a callback run by the `IOLoop` + throws an exception. + + By default simply logs the exception as an error. Subclasses + may override this method to customize reporting of exceptions. + + The exception itself is not passed explicitly, but is available + in `sys.exc_info`. + """ + app_log.error("Exception in callback %r", callback, exc_info=True) + + +class PollIOLoop(IOLoop): + """Base class for IOLoops built around a select-like function. + + For concrete implementations, see `tornado.platform.epoll.EPollIOLoop` + (Linux), `tornado.platform.kqueue.KQueueIOLoop` (BSD and Mac), or + `tornado.platform.select.SelectIOLoop` (all platforms). + """ + def initialize(self, impl, time_func=None): + super(PollIOLoop, self).initialize() + self._impl = impl + if hasattr(self._impl, 'fileno'): + set_close_exec(self._impl.fileno()) + self.time_func = time_func or time.time + self._handlers = {} + self._events = {} + self._callbacks = [] + self._callback_lock = threading.Lock() + self._timeouts = [] + self._cancellations = 0 + self._running = False + self._stopped = False + self._closing = False + self._thread_ident = None + self._blocking_signal_threshold = None + + # Create a pipe that we send bogus data to when we want to wake + # the I/O loop when it is idle + self._waker = Waker() + self.add_handler(self._waker.fileno(), + lambda fd, events: self._waker.consume(), + self.READ) + + def close(self, all_fds=False): + with self._callback_lock: + self._closing = True + self.remove_handler(self._waker.fileno()) + if all_fds: + for fd in self._handlers.keys(): + try: + close_method = getattr(fd, 'close', None) + if close_method is not None: + close_method() + else: + os.close(fd) + except Exception: + gen_log.debug("error closing fd %s", fd, exc_info=True) + self._waker.close() + self._impl.close() + + def add_handler(self, fd, handler, events): + self._handlers[fd] = stack_context.wrap(handler) + self._impl.register(fd, events | self.ERROR) + + def update_handler(self, fd, events): + self._impl.modify(fd, events | self.ERROR) + + def remove_handler(self, fd): + self._handlers.pop(fd, None) + self._events.pop(fd, None) + try: + self._impl.unregister(fd) + except Exception: + gen_log.debug("Error deleting fd from IOLoop", exc_info=True) + + def set_blocking_signal_threshold(self, seconds, action): + if not hasattr(signal, "setitimer"): + gen_log.error("set_blocking_signal_threshold requires a signal module " + "with the setitimer method") + return + self._blocking_signal_threshold = seconds + if seconds is not None: + signal.signal(signal.SIGALRM, + action if action is not None else signal.SIG_DFL) + + def start(self): + if not logging.getLogger().handlers: + # The IOLoop catches and logs exceptions, so it's + # important that log output be visible. However, python's + # default behavior for non-root loggers (prior to python + # 3.2) is to print an unhelpful "no handlers could be + # found" message rather than the actual log entry, so we + # must explicitly configure logging if we've made it this + # far without anything. + logging.basicConfig() + if self._stopped: + self._stopped = False + return + old_current = getattr(IOLoop._current, "instance", None) + IOLoop._current.instance = self + self._thread_ident = thread.get_ident() + self._running = True + + # signal.set_wakeup_fd closes a race condition in event loops: + # a signal may arrive at the beginning of select/poll/etc + # before it goes into its interruptible sleep, so the signal + # will be consumed without waking the select. The solution is + # for the (C, synchronous) signal handler to write to a pipe, + # which will then be seen by select. + # + # In python's signal handling semantics, this only matters on the + # main thread (fortunately, set_wakeup_fd only works on the main + # thread and will raise a ValueError otherwise). + # + # If someone has already set a wakeup fd, we don't want to + # disturb it. This is an issue for twisted, which does its + # SIGCHILD processing in response to its own wakeup fd being + # written to. As long as the wakeup fd is registered on the IOLoop, + # the loop will still wake up and everything should work. + old_wakeup_fd = None + if hasattr(signal, 'set_wakeup_fd') and os.name == 'posix': + # requires python 2.6+, unix. set_wakeup_fd exists but crashes + # the python process on windows. + try: + old_wakeup_fd = signal.set_wakeup_fd(self._waker.write_fileno()) + if old_wakeup_fd != -1: + # Already set, restore previous value. This is a little racy, + # but there's no clean get_wakeup_fd and in real use the + # IOLoop is just started once at the beginning. + signal.set_wakeup_fd(old_wakeup_fd) + old_wakeup_fd = None + except ValueError: # non-main thread + pass + + while True: + poll_timeout = 3600.0 + + # Prevent IO event starvation by delaying new callbacks + # to the next iteration of the event loop. + with self._callback_lock: + callbacks = self._callbacks + self._callbacks = [] + for callback in callbacks: + self._run_callback(callback) + + if self._timeouts: + now = self.time() + while self._timeouts: + if self._timeouts[0].callback is None: + # the timeout was cancelled + heapq.heappop(self._timeouts) + self._cancellations -= 1 + elif self._timeouts[0].deadline <= now: + timeout = heapq.heappop(self._timeouts) + self._run_callback(timeout.callback) + else: + seconds = self._timeouts[0].deadline - now + poll_timeout = min(seconds, poll_timeout) + break + if (self._cancellations > 512 + and self._cancellations > (len(self._timeouts) >> 1)): + # Clean up the timeout queue when it gets large and it's + # more than half cancellations. + self._cancellations = 0 + self._timeouts = [x for x in self._timeouts + if x.callback is not None] + heapq.heapify(self._timeouts) + + if self._callbacks: + # If any callbacks or timeouts called add_callback, + # we don't want to wait in poll() before we run them. + poll_timeout = 0.0 + + if not self._running: + break + + if self._blocking_signal_threshold is not None: + # clear alarm so it doesn't fire while poll is waiting for + # events. + signal.setitimer(signal.ITIMER_REAL, 0, 0) + + try: + event_pairs = self._impl.poll(poll_timeout) + except Exception as e: + # Depending on python version and IOLoop implementation, + # different exception types may be thrown and there are + # two ways EINTR might be signaled: + # * e.errno == errno.EINTR + # * e.args is like (errno.EINTR, 'Interrupted system call') + if (getattr(e, 'errno', None) == errno.EINTR or + (isinstance(getattr(e, 'args', None), tuple) and + len(e.args) == 2 and e.args[0] == errno.EINTR)): + continue + else: + raise + + if self._blocking_signal_threshold is not None: + signal.setitimer(signal.ITIMER_REAL, + self._blocking_signal_threshold, 0) + + # Pop one fd at a time from the set of pending fds and run + # its handler. Since that handler may perform actions on + # other file descriptors, there may be reentrant calls to + # this IOLoop that update self._events + self._events.update(event_pairs) + while self._events: + fd, events = self._events.popitem() + try: + self._handlers[fd](fd, events) + except (OSError, IOError) as e: + if e.args[0] == errno.EPIPE: + # Happens when the client closes the connection + pass + else: + app_log.error("Exception in I/O handler for fd %s", + fd, exc_info=True) + except Exception: + app_log.error("Exception in I/O handler for fd %s", + fd, exc_info=True) + # reset the stopped flag so another start/stop pair can be issued + self._stopped = False + if self._blocking_signal_threshold is not None: + signal.setitimer(signal.ITIMER_REAL, 0, 0) + IOLoop._current.instance = old_current + if old_wakeup_fd is not None: + signal.set_wakeup_fd(old_wakeup_fd) + + def stop(self): + self._running = False + self._stopped = True + self._waker.wake() + + def time(self): + return self.time_func() + + def add_timeout(self, deadline, callback): + timeout = _Timeout(deadline, stack_context.wrap(callback), self) + heapq.heappush(self._timeouts, timeout) + return timeout + + def remove_timeout(self, timeout): + # Removing from a heap is complicated, so just leave the defunct + # timeout object in the queue (see discussion in + # http://docs.python.org/library/heapq.html). + # If this turns out to be a problem, we could add a garbage + # collection pass whenever there are too many dead timeouts. + timeout.callback = None + self._cancellations += 1 + + def add_callback(self, callback, *args, **kwargs): + with self._callback_lock: + if self._closing: + raise RuntimeError("IOLoop is closing") + list_empty = not self._callbacks + self._callbacks.append(functools.partial( + stack_context.wrap(callback), *args, **kwargs)) + if list_empty and thread.get_ident() != self._thread_ident: + # If we're in the IOLoop's thread, we know it's not currently + # polling. If we're not, and we added the first callback to an + # empty list, we may need to wake it up (it may wake up on its + # own, but an occasional extra wake is harmless). Waking + # up a polling IOLoop is relatively expensive, so we try to + # avoid it when we can. + self._waker.wake() + + def add_callback_from_signal(self, callback, *args, **kwargs): + with stack_context.NullContext(): + if thread.get_ident() != self._thread_ident: + # if the signal is handled on another thread, we can add + # it normally (modulo the NullContext) + self.add_callback(callback, *args, **kwargs) + else: + # If we're on the IOLoop's thread, we cannot use + # the regular add_callback because it may deadlock on + # _callback_lock. Blindly insert into self._callbacks. + # This is safe because the GIL makes list.append atomic. + # One subtlety is that if the signal interrupted the + # _callback_lock block in IOLoop.start, we may modify + # either the old or new version of self._callbacks, + # but either way will work. + self._callbacks.append(functools.partial( + stack_context.wrap(callback), *args, **kwargs)) + + +class _Timeout(object): + """An IOLoop timeout, a UNIX timestamp and a callback""" + + # Reduce memory overhead when there are lots of pending callbacks + __slots__ = ['deadline', 'callback'] + + def __init__(self, deadline, callback, io_loop): + if isinstance(deadline, numbers.Real): + self.deadline = deadline + elif isinstance(deadline, datetime.timedelta): + self.deadline = io_loop.time() + _Timeout.timedelta_to_seconds(deadline) + else: + raise TypeError("Unsupported deadline %r" % deadline) + self.callback = callback + + @staticmethod + def timedelta_to_seconds(td): + """Equivalent to td.total_seconds() (introduced in python 2.7).""" + return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6) + + # Comparison methods to sort by deadline, with object id as a tiebreaker + # to guarantee a consistent ordering. The heapq module uses __le__ + # in python2.5, and __lt__ in 2.6+ (sort() and most other comparisons + # use __lt__). + def __lt__(self, other): + return ((self.deadline, id(self)) < + (other.deadline, id(other))) + + def __le__(self, other): + return ((self.deadline, id(self)) <= + (other.deadline, id(other))) + + +class PeriodicCallback(object): + """Schedules the given callback to be called periodically. + + The callback is called every ``callback_time`` milliseconds. + + `start` must be called after the `PeriodicCallback` is created. + """ + def __init__(self, callback, callback_time, io_loop=None): + self.callback = callback + if callback_time <= 0: + raise ValueError("Periodic callback must have a positive callback_time") + self.callback_time = callback_time + self.io_loop = io_loop or IOLoop.current() + self._running = False + self._timeout = None + + def start(self): + """Starts the timer.""" + self._running = True + self._next_timeout = self.io_loop.time() + self._schedule_next() + + def stop(self): + """Stops the timer.""" + self._running = False + if self._timeout is not None: + self.io_loop.remove_timeout(self._timeout) + self._timeout = None + + def _run(self): + if not self._running: + return + try: + self.callback() + except Exception: + app_log.error("Error in periodic callback", exc_info=True) + self._schedule_next() + + def _schedule_next(self): + if self._running: + current_time = self.io_loop.time() + while self._next_timeout <= current_time: + self._next_timeout += self.callback_time / 1000.0 + self._timeout = self.io_loop.add_timeout(self._next_timeout, self._run) diff --git a/scripts/external_libs/zmq/eventloop/minitornado/log.py b/scripts/external_libs/zmq/eventloop/minitornado/log.py new file mode 100644 index 00000000..49051e89 --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/log.py @@ -0,0 +1,6 @@ +"""minimal subset of tornado.log for zmq.eventloop.minitornado""" + +import logging + +app_log = logging.getLogger("tornado.application") +gen_log = logging.getLogger("tornado.general") diff --git a/scripts/external_libs/zmq/eventloop/minitornado/platform/__init__.py b/scripts/external_libs/zmq/eventloop/minitornado/platform/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/platform/__init__.py diff --git a/scripts/external_libs/zmq/eventloop/minitornado/platform/auto.py b/scripts/external_libs/zmq/eventloop/minitornado/platform/auto.py new file mode 100644 index 00000000..b40ccd94 --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/platform/auto.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# +# Copyright 2011 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Implementation of platform-specific functionality. + +For each function or class described in `tornado.platform.interface`, +the appropriate platform-specific implementation exists in this module. +Most code that needs access to this functionality should do e.g.:: + + from tornado.platform.auto import set_close_exec +""" + +from __future__ import absolute_import, division, print_function, with_statement + +import os + +if os.name == 'nt': + from .common import Waker + from .windows import set_close_exec +else: + from .posix import set_close_exec, Waker + +try: + # monotime monkey-patches the time module to have a monotonic function + # in versions of python before 3.3. + import monotime +except ImportError: + pass +try: + from time import monotonic as monotonic_time +except ImportError: + monotonic_time = None diff --git a/scripts/external_libs/zmq/eventloop/minitornado/platform/common.py b/scripts/external_libs/zmq/eventloop/minitornado/platform/common.py new file mode 100644 index 00000000..2d75dc1e --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/platform/common.py @@ -0,0 +1,91 @@ +"""Lowest-common-denominator implementations of platform functionality.""" +from __future__ import absolute_import, division, print_function, with_statement + +import errno +import socket + +from . import interface + + +class Waker(interface.Waker): + """Create an OS independent asynchronous pipe. + + For use on platforms that don't have os.pipe() (or where pipes cannot + be passed to select()), but do have sockets. This includes Windows + and Jython. + """ + def __init__(self): + # Based on Zope async.py: http://svn.zope.org/zc.ngi/trunk/src/zc/ngi/async.py + + self.writer = socket.socket() + # Disable buffering -- pulling the trigger sends 1 byte, + # and we want that sent immediately, to wake up ASAP. + self.writer.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + count = 0 + while 1: + count += 1 + # Bind to a local port; for efficiency, let the OS pick + # a free port for us. + # Unfortunately, stress tests showed that we may not + # be able to connect to that port ("Address already in + # use") despite that the OS picked it. This appears + # to be a race bug in the Windows socket implementation. + # So we loop until a connect() succeeds (almost always + # on the first try). See the long thread at + # http://mail.zope.org/pipermail/zope/2005-July/160433.html + # for hideous details. + a = socket.socket() + a.bind(("127.0.0.1", 0)) + a.listen(1) + connect_address = a.getsockname() # assigned (host, port) pair + try: + self.writer.connect(connect_address) + break # success + except socket.error as detail: + if (not hasattr(errno, 'WSAEADDRINUSE') or + detail[0] != errno.WSAEADDRINUSE): + # "Address already in use" is the only error + # I've seen on two WinXP Pro SP2 boxes, under + # Pythons 2.3.5 and 2.4.1. + raise + # (10048, 'Address already in use') + # assert count <= 2 # never triggered in Tim's tests + if count >= 10: # I've never seen it go above 2 + a.close() + self.writer.close() + raise socket.error("Cannot bind trigger!") + # Close `a` and try again. Note: I originally put a short + # sleep() here, but it didn't appear to help or hurt. + a.close() + + self.reader, addr = a.accept() + self.reader.setblocking(0) + self.writer.setblocking(0) + a.close() + self.reader_fd = self.reader.fileno() + + def fileno(self): + return self.reader.fileno() + + def write_fileno(self): + return self.writer.fileno() + + def wake(self): + try: + self.writer.send(b"x") + except (IOError, socket.error): + pass + + def consume(self): + try: + while True: + result = self.reader.recv(1024) + if not result: + break + except (IOError, socket.error): + pass + + def close(self): + self.reader.close() + self.writer.close() diff --git a/scripts/external_libs/zmq/eventloop/minitornado/platform/interface.py b/scripts/external_libs/zmq/eventloop/minitornado/platform/interface.py new file mode 100644 index 00000000..07da6bab --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/platform/interface.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# +# Copyright 2011 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Interfaces for platform-specific functionality. + +This module exists primarily for documentation purposes and as base classes +for other tornado.platform modules. Most code should import the appropriate +implementation from `tornado.platform.auto`. +""" + +from __future__ import absolute_import, division, print_function, with_statement + + +def set_close_exec(fd): + """Sets the close-on-exec bit (``FD_CLOEXEC``)for a file descriptor.""" + raise NotImplementedError() + + +class Waker(object): + """A socket-like object that can wake another thread from ``select()``. + + The `~tornado.ioloop.IOLoop` will add the Waker's `fileno()` to + its ``select`` (or ``epoll`` or ``kqueue``) calls. When another + thread wants to wake up the loop, it calls `wake`. Once it has woken + up, it will call `consume` to do any necessary per-wake cleanup. When + the ``IOLoop`` is closed, it closes its waker too. + """ + def fileno(self): + """Returns the read file descriptor for this waker. + + Must be suitable for use with ``select()`` or equivalent on the + local platform. + """ + raise NotImplementedError() + + def write_fileno(self): + """Returns the write file descriptor for this waker.""" + raise NotImplementedError() + + def wake(self): + """Triggers activity on the waker's file descriptor.""" + raise NotImplementedError() + + def consume(self): + """Called after the listen has woken up to do any necessary cleanup.""" + raise NotImplementedError() + + def close(self): + """Closes the waker's file descriptor(s).""" + raise NotImplementedError() diff --git a/scripts/external_libs/zmq/eventloop/minitornado/platform/posix.py b/scripts/external_libs/zmq/eventloop/minitornado/platform/posix.py new file mode 100644 index 00000000..ccffbb66 --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/platform/posix.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Copyright 2011 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Posix implementations of platform-specific functionality.""" + +from __future__ import absolute_import, division, print_function, with_statement + +import fcntl +import os + +from . import interface + + +def set_close_exec(fd): + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + + +def _set_nonblocking(fd): + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + + +class Waker(interface.Waker): + def __init__(self): + r, w = os.pipe() + _set_nonblocking(r) + _set_nonblocking(w) + set_close_exec(r) + set_close_exec(w) + self.reader = os.fdopen(r, "rb", 0) + self.writer = os.fdopen(w, "wb", 0) + + def fileno(self): + return self.reader.fileno() + + def write_fileno(self): + return self.writer.fileno() + + def wake(self): + try: + self.writer.write(b"x") + except IOError: + pass + + def consume(self): + try: + while True: + result = self.reader.read() + if not result: + break + except IOError: + pass + + def close(self): + self.reader.close() + self.writer.close() diff --git a/scripts/external_libs/zmq/eventloop/minitornado/platform/windows.py b/scripts/external_libs/zmq/eventloop/minitornado/platform/windows.py new file mode 100644 index 00000000..817bdca1 --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/platform/windows.py @@ -0,0 +1,20 @@ +# NOTE: win32 support is currently experimental, and not recommended +# for production use. + + +from __future__ import absolute_import, division, print_function, with_statement +import ctypes +import ctypes.wintypes + +# See: http://msdn.microsoft.com/en-us/library/ms724935(VS.85).aspx +SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation +SetHandleInformation.argtypes = (ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD, ctypes.wintypes.DWORD) +SetHandleInformation.restype = ctypes.wintypes.BOOL + +HANDLE_FLAG_INHERIT = 0x00000001 + + +def set_close_exec(fd): + success = SetHandleInformation(fd, HANDLE_FLAG_INHERIT, 0) + if not success: + raise ctypes.GetLastError() diff --git a/scripts/external_libs/zmq/eventloop/minitornado/stack_context.py b/scripts/external_libs/zmq/eventloop/minitornado/stack_context.py new file mode 100644 index 00000000..226d8042 --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/stack_context.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python +# +# Copyright 2010 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""`StackContext` allows applications to maintain threadlocal-like state +that follows execution as it moves to other execution contexts. + +The motivating examples are to eliminate the need for explicit +``async_callback`` wrappers (as in `tornado.web.RequestHandler`), and to +allow some additional context to be kept for logging. + +This is slightly magic, but it's an extension of the idea that an +exception handler is a kind of stack-local state and when that stack +is suspended and resumed in a new context that state needs to be +preserved. `StackContext` shifts the burden of restoring that state +from each call site (e.g. wrapping each `.AsyncHTTPClient` callback +in ``async_callback``) to the mechanisms that transfer control from +one context to another (e.g. `.AsyncHTTPClient` itself, `.IOLoop`, +thread pools, etc). + +Example usage:: + + @contextlib.contextmanager + def die_on_error(): + try: + yield + except Exception: + logging.error("exception in asynchronous operation",exc_info=True) + sys.exit(1) + + with StackContext(die_on_error): + # Any exception thrown here *or in callback and its desendents* + # will cause the process to exit instead of spinning endlessly + # in the ioloop. + http_client.fetch(url, callback) + ioloop.start() + +Most applications shouln't have to work with `StackContext` directly. +Here are a few rules of thumb for when it's necessary: + +* If you're writing an asynchronous library that doesn't rely on a + stack_context-aware library like `tornado.ioloop` or `tornado.iostream` + (for example, if you're writing a thread pool), use + `.stack_context.wrap()` before any asynchronous operations to capture the + stack context from where the operation was started. + +* If you're writing an asynchronous library that has some shared + resources (such as a connection pool), create those shared resources + within a ``with stack_context.NullContext():`` block. This will prevent + ``StackContexts`` from leaking from one request to another. + +* If you want to write something like an exception handler that will + persist across asynchronous calls, create a new `StackContext` (or + `ExceptionStackContext`), and make your asynchronous calls in a ``with`` + block that references your `StackContext`. +""" + +from __future__ import absolute_import, division, print_function, with_statement + +import sys +import threading + +from .util import raise_exc_info + + +class StackContextInconsistentError(Exception): + pass + + +class _State(threading.local): + def __init__(self): + self.contexts = (tuple(), None) +_state = _State() + + +class StackContext(object): + """Establishes the given context as a StackContext that will be transferred. + + Note that the parameter is a callable that returns a context + manager, not the context itself. That is, where for a + non-transferable context manager you would say:: + + with my_context(): + + StackContext takes the function itself rather than its result:: + + with StackContext(my_context): + + The result of ``with StackContext() as cb:`` is a deactivation + callback. Run this callback when the StackContext is no longer + needed to ensure that it is not propagated any further (note that + deactivating a context does not affect any instances of that + context that are currently pending). This is an advanced feature + and not necessary in most applications. + """ + def __init__(self, context_factory): + self.context_factory = context_factory + self.contexts = [] + self.active = True + + def _deactivate(self): + self.active = False + + # StackContext protocol + def enter(self): + context = self.context_factory() + self.contexts.append(context) + context.__enter__() + + def exit(self, type, value, traceback): + context = self.contexts.pop() + context.__exit__(type, value, traceback) + + # Note that some of this code is duplicated in ExceptionStackContext + # below. ExceptionStackContext is more common and doesn't need + # the full generality of this class. + def __enter__(self): + self.old_contexts = _state.contexts + self.new_contexts = (self.old_contexts[0] + (self,), self) + _state.contexts = self.new_contexts + + try: + self.enter() + except: + _state.contexts = self.old_contexts + raise + + return self._deactivate + + def __exit__(self, type, value, traceback): + try: + self.exit(type, value, traceback) + finally: + final_contexts = _state.contexts + _state.contexts = self.old_contexts + + # Generator coroutines and with-statements with non-local + # effects interact badly. Check here for signs of + # the stack getting out of sync. + # Note that this check comes after restoring _state.context + # so that if it fails things are left in a (relatively) + # consistent state. + if final_contexts is not self.new_contexts: + raise StackContextInconsistentError( + 'stack_context inconsistency (may be caused by yield ' + 'within a "with StackContext" block)') + + # Break up a reference to itself to allow for faster GC on CPython. + self.new_contexts = None + + +class ExceptionStackContext(object): + """Specialization of StackContext for exception handling. + + The supplied ``exception_handler`` function will be called in the + event of an uncaught exception in this context. The semantics are + similar to a try/finally clause, and intended use cases are to log + an error, close a socket, or similar cleanup actions. The + ``exc_info`` triple ``(type, value, traceback)`` will be passed to the + exception_handler function. + + If the exception handler returns true, the exception will be + consumed and will not be propagated to other exception handlers. + """ + def __init__(self, exception_handler): + self.exception_handler = exception_handler + self.active = True + + def _deactivate(self): + self.active = False + + def exit(self, type, value, traceback): + if type is not None: + return self.exception_handler(type, value, traceback) + + def __enter__(self): + self.old_contexts = _state.contexts + self.new_contexts = (self.old_contexts[0], self) + _state.contexts = self.new_contexts + + return self._deactivate + + def __exit__(self, type, value, traceback): + try: + if type is not None: + return self.exception_handler(type, value, traceback) + finally: + final_contexts = _state.contexts + _state.contexts = self.old_contexts + + if final_contexts is not self.new_contexts: + raise StackContextInconsistentError( + 'stack_context inconsistency (may be caused by yield ' + 'within a "with StackContext" block)') + + # Break up a reference to itself to allow for faster GC on CPython. + self.new_contexts = None + + +class NullContext(object): + """Resets the `StackContext`. + + Useful when creating a shared resource on demand (e.g. an + `.AsyncHTTPClient`) where the stack that caused the creating is + not relevant to future operations. + """ + def __enter__(self): + self.old_contexts = _state.contexts + _state.contexts = (tuple(), None) + + def __exit__(self, type, value, traceback): + _state.contexts = self.old_contexts + + +def _remove_deactivated(contexts): + """Remove deactivated handlers from the chain""" + # Clean ctx handlers + stack_contexts = tuple([h for h in contexts[0] if h.active]) + + # Find new head + head = contexts[1] + while head is not None and not head.active: + head = head.old_contexts[1] + + # Process chain + ctx = head + while ctx is not None: + parent = ctx.old_contexts[1] + + while parent is not None: + if parent.active: + break + ctx.old_contexts = parent.old_contexts + parent = parent.old_contexts[1] + + ctx = parent + + return (stack_contexts, head) + + +def wrap(fn): + """Returns a callable object that will restore the current `StackContext` + when executed. + + Use this whenever saving a callback to be executed later in a + different execution context (either in a different thread or + asynchronously in the same thread). + """ + # Check if function is already wrapped + if fn is None or hasattr(fn, '_wrapped'): + return fn + + # Capture current stack head + # TODO: Any other better way to store contexts and update them in wrapped function? + cap_contexts = [_state.contexts] + + def wrapped(*args, **kwargs): + ret = None + try: + # Capture old state + current_state = _state.contexts + + # Remove deactivated items + cap_contexts[0] = contexts = _remove_deactivated(cap_contexts[0]) + + # Force new state + _state.contexts = contexts + + # Current exception + exc = (None, None, None) + top = None + + # Apply stack contexts + last_ctx = 0 + stack = contexts[0] + + # Apply state + for n in stack: + try: + n.enter() + last_ctx += 1 + except: + # Exception happened. Record exception info and store top-most handler + exc = sys.exc_info() + top = n.old_contexts[1] + + # Execute callback if no exception happened while restoring state + if top is None: + try: + ret = fn(*args, **kwargs) + except: + exc = sys.exc_info() + top = contexts[1] + + # If there was exception, try to handle it by going through the exception chain + if top is not None: + exc = _handle_exception(top, exc) + else: + # Otherwise take shorter path and run stack contexts in reverse order + while last_ctx > 0: + last_ctx -= 1 + c = stack[last_ctx] + + try: + c.exit(*exc) + except: + exc = sys.exc_info() + top = c.old_contexts[1] + break + else: + top = None + + # If if exception happened while unrolling, take longer exception handler path + if top is not None: + exc = _handle_exception(top, exc) + + # If exception was not handled, raise it + if exc != (None, None, None): + raise_exc_info(exc) + finally: + _state.contexts = current_state + return ret + + wrapped._wrapped = True + return wrapped + + +def _handle_exception(tail, exc): + while tail is not None: + try: + if tail.exit(*exc): + exc = (None, None, None) + except: + exc = sys.exc_info() + + tail = tail.old_contexts[1] + + return exc + + +def run_with_stack_context(context, func): + """Run a coroutine ``func`` in the given `StackContext`. + + It is not safe to have a ``yield`` statement within a ``with StackContext`` + block, so it is difficult to use stack context with `.gen.coroutine`. + This helper function runs the function in the correct context while + keeping the ``yield`` and ``with`` statements syntactically separate. + + Example:: + + @gen.coroutine + def incorrect(): + with StackContext(ctx): + # ERROR: this will raise StackContextInconsistentError + yield other_coroutine() + + @gen.coroutine + def correct(): + yield run_with_stack_context(StackContext(ctx), other_coroutine) + + .. versionadded:: 3.1 + """ + with context: + return func() diff --git a/scripts/external_libs/zmq/eventloop/minitornado/util.py b/scripts/external_libs/zmq/eventloop/minitornado/util.py new file mode 100644 index 00000000..c1e2eb95 --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/minitornado/util.py @@ -0,0 +1,184 @@ +"""Miscellaneous utility functions and classes. + +This module is used internally by Tornado. It is not necessarily expected +that the functions and classes defined here will be useful to other +applications, but they are documented here in case they are. + +The one public-facing part of this module is the `Configurable` class +and its `~Configurable.configure` method, which becomes a part of the +interface of its subclasses, including `.AsyncHTTPClient`, `.IOLoop`, +and `.Resolver`. +""" + +from __future__ import absolute_import, division, print_function, with_statement + +import sys + + +def import_object(name): + """Imports an object by name. + + import_object('x') is equivalent to 'import x'. + import_object('x.y.z') is equivalent to 'from x.y import z'. + + >>> import tornado.escape + >>> import_object('tornado.escape') is tornado.escape + True + >>> import_object('tornado.escape.utf8') is tornado.escape.utf8 + True + >>> import_object('tornado') is tornado + True + >>> import_object('tornado.missing_module') + Traceback (most recent call last): + ... + ImportError: No module named missing_module + """ + if name.count('.') == 0: + return __import__(name, None, None) + + parts = name.split('.') + obj = __import__('.'.join(parts[:-1]), None, None, [parts[-1]], 0) + try: + return getattr(obj, parts[-1]) + except AttributeError: + raise ImportError("No module named %s" % parts[-1]) + + +# Fake unicode literal support: Python 3.2 doesn't have the u'' marker for +# literal strings, and alternative solutions like "from __future__ import +# unicode_literals" have other problems (see PEP 414). u() can be applied +# to ascii strings that include \u escapes (but they must not contain +# literal non-ascii characters). +if type('') is not type(b''): + def u(s): + return s + bytes_type = bytes + unicode_type = str + basestring_type = str +else: + def u(s): + return s.decode('unicode_escape') + bytes_type = str + unicode_type = unicode + basestring_type = basestring + + +if sys.version_info > (3,): + exec(""" +def raise_exc_info(exc_info): + raise exc_info[1].with_traceback(exc_info[2]) + +def exec_in(code, glob, loc=None): + if isinstance(code, str): + code = compile(code, '<string>', 'exec', dont_inherit=True) + exec(code, glob, loc) +""") +else: + exec(""" +def raise_exc_info(exc_info): + raise exc_info[0], exc_info[1], exc_info[2] + +def exec_in(code, glob, loc=None): + if isinstance(code, basestring): + # exec(string) inherits the caller's future imports; compile + # the string first to prevent that. + code = compile(code, '<string>', 'exec', dont_inherit=True) + exec code in glob, loc +""") + + +class Configurable(object): + """Base class for configurable interfaces. + + A configurable interface is an (abstract) class whose constructor + acts as a factory function for one of its implementation subclasses. + The implementation subclass as well as optional keyword arguments to + its initializer can be set globally at runtime with `configure`. + + By using the constructor as the factory method, the interface + looks like a normal class, `isinstance` works as usual, etc. This + pattern is most useful when the choice of implementation is likely + to be a global decision (e.g. when `~select.epoll` is available, + always use it instead of `~select.select`), or when a + previously-monolithic class has been split into specialized + subclasses. + + Configurable subclasses must define the class methods + `configurable_base` and `configurable_default`, and use the instance + method `initialize` instead of ``__init__``. + """ + __impl_class = None + __impl_kwargs = None + + def __new__(cls, **kwargs): + base = cls.configurable_base() + args = {} + if cls is base: + impl = cls.configured_class() + if base.__impl_kwargs: + args.update(base.__impl_kwargs) + else: + impl = cls + args.update(kwargs) + instance = super(Configurable, cls).__new__(impl) + # initialize vs __init__ chosen for compatiblity with AsyncHTTPClient + # singleton magic. If we get rid of that we can switch to __init__ + # here too. + instance.initialize(**args) + return instance + + @classmethod + def configurable_base(cls): + """Returns the base class of a configurable hierarchy. + + This will normally return the class in which it is defined. + (which is *not* necessarily the same as the cls classmethod parameter). + """ + raise NotImplementedError() + + @classmethod + def configurable_default(cls): + """Returns the implementation class to be used if none is configured.""" + raise NotImplementedError() + + def initialize(self): + """Initialize a `Configurable` subclass instance. + + Configurable classes should use `initialize` instead of ``__init__``. + """ + + @classmethod + def configure(cls, impl, **kwargs): + """Sets the class to use when the base class is instantiated. + + Keyword arguments will be saved and added to the arguments passed + to the constructor. This can be used to set global defaults for + some parameters. + """ + base = cls.configurable_base() + if isinstance(impl, (unicode_type, bytes_type)): + impl = import_object(impl) + if impl is not None and not issubclass(impl, cls): + raise ValueError("Invalid subclass of %s" % cls) + base.__impl_class = impl + base.__impl_kwargs = kwargs + + @classmethod + def configured_class(cls): + """Returns the currently configured class.""" + base = cls.configurable_base() + if cls.__impl_class is None: + base.__impl_class = cls.configurable_default() + return base.__impl_class + + @classmethod + def _save_configuration(cls): + base = cls.configurable_base() + return (base.__impl_class, base.__impl_kwargs) + + @classmethod + def _restore_configuration(cls, saved): + base = cls.configurable_base() + base.__impl_class = saved[0] + base.__impl_kwargs = saved[1] + diff --git a/scripts/external_libs/zmq/eventloop/zmqstream.py b/scripts/external_libs/zmq/eventloop/zmqstream.py new file mode 100644 index 00000000..86a97e44 --- /dev/null +++ b/scripts/external_libs/zmq/eventloop/zmqstream.py @@ -0,0 +1,529 @@ +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A utility class to send to and recv from a non-blocking socket.""" + +from __future__ import with_statement + +import sys + +import zmq +from zmq.utils import jsonapi + +try: + import cPickle as pickle +except ImportError: + import pickle + +from .ioloop import IOLoop + +try: + # gen_log will only import from >= 3.0 + from tornado.log import gen_log + from tornado import stack_context +except ImportError: + from .minitornado.log import gen_log + from .minitornado import stack_context + +try: + from queue import Queue +except ImportError: + from Queue import Queue + +from zmq.utils.strtypes import bytes, unicode, basestring + +try: + callable +except NameError: + callable = lambda obj: hasattr(obj, '__call__') + + +class ZMQStream(object): + """A utility class to register callbacks when a zmq socket sends and receives + + For use with zmq.eventloop.ioloop + + There are three main methods + + Methods: + + * **on_recv(callback, copy=True):** + register a callback to be run every time the socket has something to receive + * **on_send(callback):** + register a callback to be run every time you call send + * **send(self, msg, flags=0, copy=False, callback=None):** + perform a send that will trigger the callback + if callback is passed, on_send is also called. + + There are also send_multipart(), send_json(), send_pyobj() + + Three other methods for deactivating the callbacks: + + * **stop_on_recv():** + turn off the recv callback + * **stop_on_send():** + turn off the send callback + + which simply call ``on_<evt>(None)``. + + The entire socket interface, excluding direct recv methods, is also + provided, primarily through direct-linking the methods. + e.g. + + >>> stream.bind is stream.socket.bind + True + + """ + + socket = None + io_loop = None + poller = None + + def __init__(self, socket, io_loop=None): + self.socket = socket + self.io_loop = io_loop or IOLoop.instance() + self.poller = zmq.Poller() + + self._send_queue = Queue() + self._recv_callback = None + self._send_callback = None + self._close_callback = None + self._recv_copy = False + self._flushed = False + + self._state = self.io_loop.ERROR + self._init_io_state() + + # shortcircuit some socket methods + self.bind = self.socket.bind + self.bind_to_random_port = self.socket.bind_to_random_port + self.connect = self.socket.connect + self.setsockopt = self.socket.setsockopt + self.getsockopt = self.socket.getsockopt + self.setsockopt_string = self.socket.setsockopt_string + self.getsockopt_string = self.socket.getsockopt_string + self.setsockopt_unicode = self.socket.setsockopt_unicode + self.getsockopt_unicode = self.socket.getsockopt_unicode + + + def stop_on_recv(self): + """Disable callback and automatic receiving.""" + return self.on_recv(None) + + def stop_on_send(self): + """Disable callback on sending.""" + return self.on_send(None) + + def stop_on_err(self): + """DEPRECATED, does nothing""" + gen_log.warn("on_err does nothing, and will be removed") + + def on_err(self, callback): + """DEPRECATED, does nothing""" + gen_log.warn("on_err does nothing, and will be removed") + + def on_recv(self, callback, copy=True): + """Register a callback for when a message is ready to recv. + + There can be only one callback registered at a time, so each + call to `on_recv` replaces previously registered callbacks. + + on_recv(None) disables recv event polling. + + Use on_recv_stream(callback) instead, to register a callback that will receive + both this ZMQStream and the message, instead of just the message. + + Parameters + ---------- + + callback : callable + callback must take exactly one argument, which will be a + list, as returned by socket.recv_multipart() + if callback is None, recv callbacks are disabled. + copy : bool + copy is passed directly to recv, so if copy is False, + callback will receive Message objects. If copy is True, + then callback will receive bytes/str objects. + + Returns : None + """ + + self._check_closed() + assert callback is None or callable(callback) + self._recv_callback = stack_context.wrap(callback) + self._recv_copy = copy + if callback is None: + self._drop_io_state(self.io_loop.READ) + else: + self._add_io_state(self.io_loop.READ) + + def on_recv_stream(self, callback, copy=True): + """Same as on_recv, but callback will get this stream as first argument + + callback must take exactly two arguments, as it will be called as:: + + callback(stream, msg) + + Useful when a single callback should be used with multiple streams. + """ + if callback is None: + self.stop_on_recv() + else: + self.on_recv(lambda msg: callback(self, msg), copy=copy) + + def on_send(self, callback): + """Register a callback to be called on each send + + There will be two arguments:: + + callback(msg, status) + + * `msg` will be the list of sendable objects that was just sent + * `status` will be the return result of socket.send_multipart(msg) - + MessageTracker or None. + + Non-copying sends return a MessageTracker object whose + `done` attribute will be True when the send is complete. + This allows users to track when an object is safe to write to + again. + + The second argument will always be None if copy=True + on the send. + + Use on_send_stream(callback) to register a callback that will be passed + this ZMQStream as the first argument, in addition to the other two. + + on_send(None) disables recv event polling. + + Parameters + ---------- + + callback : callable + callback must take exactly two arguments, which will be + the message being sent (always a list), + and the return result of socket.send_multipart(msg) - + MessageTracker or None. + + if callback is None, send callbacks are disabled. + """ + + self._check_closed() + assert callback is None or callable(callback) + self._send_callback = stack_context.wrap(callback) + + + def on_send_stream(self, callback): + """Same as on_send, but callback will get this stream as first argument + + Callback will be passed three arguments:: + + callback(stream, msg, status) + + Useful when a single callback should be used with multiple streams. + """ + if callback is None: + self.stop_on_send() + else: + self.on_send(lambda msg, status: callback(self, msg, status)) + + + def send(self, msg, flags=0, copy=True, track=False, callback=None): + """Send a message, optionally also register a new callback for sends. + See zmq.socket.send for details. + """ + return self.send_multipart([msg], flags=flags, copy=copy, track=track, callback=callback) + + def send_multipart(self, msg, flags=0, copy=True, track=False, callback=None): + """Send a multipart message, optionally also register a new callback for sends. + See zmq.socket.send_multipart for details. + """ + kwargs = dict(flags=flags, copy=copy, track=track) + self._send_queue.put((msg, kwargs)) + callback = callback or self._send_callback + if callback is not None: + self.on_send(callback) + else: + # noop callback + self.on_send(lambda *args: None) + self._add_io_state(self.io_loop.WRITE) + + def send_string(self, u, flags=0, encoding='utf-8', callback=None): + """Send a unicode message with an encoding. + See zmq.socket.send_unicode for details. + """ + if not isinstance(u, basestring): + raise TypeError("unicode/str objects only") + return self.send(u.encode(encoding), flags=flags, callback=callback) + + send_unicode = send_string + + def send_json(self, obj, flags=0, callback=None): + """Send json-serialized version of an object. + See zmq.socket.send_json for details. + """ + if jsonapi is None: + raise ImportError('jsonlib{1,2}, json or simplejson library is required.') + else: + msg = jsonapi.dumps(obj) + return self.send(msg, flags=flags, callback=callback) + + def send_pyobj(self, obj, flags=0, protocol=-1, callback=None): + """Send a Python object as a message using pickle to serialize. + + See zmq.socket.send_json for details. + """ + msg = pickle.dumps(obj, protocol) + return self.send(msg, flags, callback=callback) + + def _finish_flush(self): + """callback for unsetting _flushed flag.""" + self._flushed = False + + def flush(self, flag=zmq.POLLIN|zmq.POLLOUT, limit=None): + """Flush pending messages. + + This method safely handles all pending incoming and/or outgoing messages, + bypassing the inner loop, passing them to the registered callbacks. + + A limit can be specified, to prevent blocking under high load. + + flush will return the first time ANY of these conditions are met: + * No more events matching the flag are pending. + * the total number of events handled reaches the limit. + + Note that if ``flag|POLLIN != 0``, recv events will be flushed even if no callback + is registered, unlike normal IOLoop operation. This allows flush to be + used to remove *and ignore* incoming messages. + + Parameters + ---------- + flag : int, default=POLLIN|POLLOUT + 0MQ poll flags. + If flag|POLLIN, recv events will be flushed. + If flag|POLLOUT, send events will be flushed. + Both flags can be set at once, which is the default. + limit : None or int, optional + The maximum number of messages to send or receive. + Both send and recv count against this limit. + + Returns + ------- + int : count of events handled (both send and recv) + """ + self._check_closed() + # unset self._flushed, so callbacks will execute, in case flush has + # already been called this iteration + already_flushed = self._flushed + self._flushed = False + # initialize counters + count = 0 + def update_flag(): + """Update the poll flag, to prevent registering POLLOUT events + if we don't have pending sends.""" + return flag & zmq.POLLIN | (self.sending() and flag & zmq.POLLOUT) + flag = update_flag() + if not flag: + # nothing to do + return 0 + self.poller.register(self.socket, flag) + events = self.poller.poll(0) + while events and (not limit or count < limit): + s,event = events[0] + if event & zmq.POLLIN: # receiving + self._handle_recv() + count += 1 + if self.socket is None: + # break if socket was closed during callback + break + if event & zmq.POLLOUT and self.sending(): + self._handle_send() + count += 1 + if self.socket is None: + # break if socket was closed during callback + break + + flag = update_flag() + if flag: + self.poller.register(self.socket, flag) + events = self.poller.poll(0) + else: + events = [] + if count: # only bypass loop if we actually flushed something + # skip send/recv callbacks this iteration + self._flushed = True + # reregister them at the end of the loop + if not already_flushed: # don't need to do it again + self.io_loop.add_callback(self._finish_flush) + elif already_flushed: + self._flushed = True + + # update ioloop poll state, which may have changed + self._rebuild_io_state() + return count + + def set_close_callback(self, callback): + """Call the given callback when the stream is closed.""" + self._close_callback = stack_context.wrap(callback) + + def close(self, linger=None): + """Close this stream.""" + if self.socket is not None: + self.io_loop.remove_handler(self.socket) + self.socket.close(linger) + self.socket = None + if self._close_callback: + self._run_callback(self._close_callback) + + def receiving(self): + """Returns True if we are currently receiving from the stream.""" + return self._recv_callback is not None + + def sending(self): + """Returns True if we are currently sending to the stream.""" + return not self._send_queue.empty() + + def closed(self): + return self.socket is None + + def _run_callback(self, callback, *args, **kwargs): + """Wrap running callbacks in try/except to allow us to + close our socket.""" + try: + # Use a NullContext to ensure that all StackContexts are run + # inside our blanket exception handler rather than outside. + with stack_context.NullContext(): + callback(*args, **kwargs) + except: + gen_log.error("Uncaught exception, closing connection.", + exc_info=True) + # Close the socket on an uncaught exception from a user callback + # (It would eventually get closed when the socket object is + # gc'd, but we don't want to rely on gc happening before we + # run out of file descriptors) + self.close() + # Re-raise the exception so that IOLoop.handle_callback_exception + # can see it and log the error + raise + + def _handle_events(self, fd, events): + """This method is the actual handler for IOLoop, that gets called whenever + an event on my socket is posted. It dispatches to _handle_recv, etc.""" + # print "handling events" + if not self.socket: + gen_log.warning("Got events for closed stream %s", fd) + return + try: + # dispatch events: + if events & IOLoop.ERROR: + gen_log.error("got POLLERR event on ZMQStream, which doesn't make sense") + return + if events & IOLoop.READ: + self._handle_recv() + if not self.socket: + return + if events & IOLoop.WRITE: + self._handle_send() + if not self.socket: + return + + # rebuild the poll state + self._rebuild_io_state() + except: + gen_log.error("Uncaught exception, closing connection.", + exc_info=True) + self.close() + raise + + def _handle_recv(self): + """Handle a recv event.""" + if self._flushed: + return + try: + msg = self.socket.recv_multipart(zmq.NOBLOCK, copy=self._recv_copy) + except zmq.ZMQError as e: + if e.errno == zmq.EAGAIN: + # state changed since poll event + pass + else: + gen_log.error("RECV Error: %s"%zmq.strerror(e.errno)) + else: + if self._recv_callback: + callback = self._recv_callback + # self._recv_callback = None + self._run_callback(callback, msg) + + # self.update_state() + + + def _handle_send(self): + """Handle a send event.""" + if self._flushed: + return + if not self.sending(): + gen_log.error("Shouldn't have handled a send event") + return + + msg, kwargs = self._send_queue.get() + try: + status = self.socket.send_multipart(msg, **kwargs) + except zmq.ZMQError as e: + gen_log.error("SEND Error: %s", e) + status = e + if self._send_callback: + callback = self._send_callback + self._run_callback(callback, msg, status) + + # self.update_state() + + def _check_closed(self): + if not self.socket: + raise IOError("Stream is closed") + + def _rebuild_io_state(self): + """rebuild io state based on self.sending() and receiving()""" + if self.socket is None: + return + state = self.io_loop.ERROR + if self.receiving(): + state |= self.io_loop.READ + if self.sending(): + state |= self.io_loop.WRITE + if state != self._state: + self._state = state + self._update_handler(state) + + def _add_io_state(self, state): + """Add io_state to poller.""" + if not self._state & state: + self._state = self._state | state + self._update_handler(self._state) + + def _drop_io_state(self, state): + """Stop poller from watching an io_state.""" + if self._state & state: + self._state = self._state & (~state) + self._update_handler(self._state) + + def _update_handler(self, state): + """Update IOLoop handler with state.""" + if self.socket is None: + return + self.io_loop.update_handler(self.socket, state) + + def _init_io_state(self): + """initialize the ioloop event handler""" + with stack_context.NullContext(): + self.io_loop.add_handler(self.socket, self._handle_events, self._state) + diff --git a/scripts/external_libs/zmq/green/__init__.py b/scripts/external_libs/zmq/green/__init__.py new file mode 100644 index 00000000..ff7e5965 --- /dev/null +++ b/scripts/external_libs/zmq/green/__init__.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +#----------------------------------------------------------------------------- +# Copyright (C) 2011-2012 Travis Cline +# +# This file is part of pyzmq +# It is adapted from upstream project zeromq_gevent under the New BSD License +# +# Distributed under the terms of the New BSD License. The full license is in +# the file COPYING.BSD, distributed as part of this software. +#----------------------------------------------------------------------------- + +"""zmq.green - gevent compatibility with zeromq. + +Usage +----- + +Instead of importing zmq directly, do so in the following manner: + +.. + + import zmq.green as zmq + + +Any calls that would have blocked the current thread will now only block the +current green thread. + +This compatibility is accomplished by ensuring the nonblocking flag is set +before any blocking operation and the ØMQ file descriptor is polled internally +to trigger needed events. +""" + +from zmq import * +from zmq.green.core import _Context, _Socket +from zmq.green.poll import _Poller +Context = _Context +Socket = _Socket +Poller = _Poller + +from zmq.green.device import device + diff --git a/scripts/external_libs/zmq/green/core.py b/scripts/external_libs/zmq/green/core.py new file mode 100644 index 00000000..9fc73e32 --- /dev/null +++ b/scripts/external_libs/zmq/green/core.py @@ -0,0 +1,287 @@ +#----------------------------------------------------------------------------- +# Copyright (C) 2011-2012 Travis Cline +# +# This file is part of pyzmq +# It is adapted from upstream project zeromq_gevent under the New BSD License +# +# Distributed under the terms of the New BSD License. The full license is in +# the file COPYING.BSD, distributed as part of this software. +#----------------------------------------------------------------------------- + +"""This module wraps the :class:`Socket` and :class:`Context` found in :mod:`pyzmq <zmq>` to be non blocking +""" + +from __future__ import print_function + +import sys +import time +import warnings + +import zmq + +from zmq import Context as _original_Context +from zmq import Socket as _original_Socket +from .poll import _Poller + +import gevent +from gevent.event import AsyncResult +from gevent.hub import get_hub + +if hasattr(zmq, 'RCVTIMEO'): + TIMEOS = (zmq.RCVTIMEO, zmq.SNDTIMEO) +else: + TIMEOS = () + +def _stop(evt): + """simple wrapper for stopping an Event, allowing for method rename in gevent 1.0""" + try: + evt.stop() + except AttributeError as e: + # gevent<1.0 compat + evt.cancel() + +class _Socket(_original_Socket): + """Green version of :class:`zmq.Socket` + + The following methods are overridden: + + * send + * recv + + To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or receiving + is deferred to the hub if a ``zmq.EAGAIN`` (retry) error is raised. + + The `__state_changed` method is triggered when the zmq.FD for the socket is + marked as readable and triggers the necessary read and write events (which + are waited for in the recv and send methods). + + Some double underscore prefixes are used to minimize pollution of + :class:`zmq.Socket`'s namespace. + """ + __in_send_multipart = False + __in_recv_multipart = False + __writable = None + __readable = None + _state_event = None + _gevent_bug_timeout = 11.6 # timeout for not trusting gevent + _debug_gevent = False # turn on if you think gevent is missing events + _poller_class = _Poller + + def __init__(self, context, socket_type): + _original_Socket.__init__(self, context, socket_type) + self.__in_send_multipart = False + self.__in_recv_multipart = False + self.__setup_events() + + + def __del__(self): + self.close() + + def close(self, linger=None): + super(_Socket, self).close(linger) + self.__cleanup_events() + + def __cleanup_events(self): + # close the _state_event event, keeps the number of active file descriptors down + if getattr(self, '_state_event', None): + _stop(self._state_event) + self._state_event = None + # if the socket has entered a close state resume any waiting greenlets + self.__writable.set() + self.__readable.set() + + def __setup_events(self): + self.__readable = AsyncResult() + self.__writable = AsyncResult() + self.__readable.set() + self.__writable.set() + + try: + self._state_event = get_hub().loop.io(self.getsockopt(zmq.FD), 1) # read state watcher + self._state_event.start(self.__state_changed) + except AttributeError: + # for gevent<1.0 compatibility + from gevent.core import read_event + self._state_event = read_event(self.getsockopt(zmq.FD), self.__state_changed, persist=True) + + def __state_changed(self, event=None, _evtype=None): + if self.closed: + self.__cleanup_events() + return + try: + # avoid triggering __state_changed from inside __state_changed + events = super(_Socket, self).getsockopt(zmq.EVENTS) + except zmq.ZMQError as exc: + self.__writable.set_exception(exc) + self.__readable.set_exception(exc) + else: + if events & zmq.POLLOUT: + self.__writable.set() + if events & zmq.POLLIN: + self.__readable.set() + + def _wait_write(self): + assert self.__writable.ready(), "Only one greenlet can be waiting on this event" + self.__writable = AsyncResult() + # timeout is because libzmq cannot be trusted to properly signal a new send event: + # this is effectively a maximum poll interval of 1s + tic = time.time() + dt = self._gevent_bug_timeout + if dt: + timeout = gevent.Timeout(seconds=dt) + else: + timeout = None + try: + if timeout: + timeout.start() + self.__writable.get(block=True) + except gevent.Timeout as t: + if t is not timeout: + raise + toc = time.time() + # gevent bug: get can raise timeout even on clean return + # don't display zmq bug warning for gevent bug (this is getting ridiculous) + if self._debug_gevent and timeout and toc-tic > dt and \ + self.getsockopt(zmq.EVENTS) & zmq.POLLOUT: + print("BUG: gevent may have missed a libzmq send event on %i!" % self.FD, file=sys.stderr) + finally: + if timeout: + timeout.cancel() + self.__writable.set() + + def _wait_read(self): + assert self.__readable.ready(), "Only one greenlet can be waiting on this event" + self.__readable = AsyncResult() + # timeout is because libzmq cannot always be trusted to play nice with libevent. + # I can only confirm that this actually happens for send, but lets be symmetrical + # with our dirty hacks. + # this is effectively a maximum poll interval of 1s + tic = time.time() + dt = self._gevent_bug_timeout + if dt: + timeout = gevent.Timeout(seconds=dt) + else: + timeout = None + try: + if timeout: + timeout.start() + self.__readable.get(block=True) + except gevent.Timeout as t: + if t is not timeout: + raise + toc = time.time() + # gevent bug: get can raise timeout even on clean return + # don't display zmq bug warning for gevent bug (this is getting ridiculous) + if self._debug_gevent and timeout and toc-tic > dt and \ + self.getsockopt(zmq.EVENTS) & zmq.POLLIN: + print("BUG: gevent may have missed a libzmq recv event on %i!" % self.FD, file=sys.stderr) + finally: + if timeout: + timeout.cancel() + self.__readable.set() + + def send(self, data, flags=0, copy=True, track=False): + """send, which will only block current greenlet + + state_changed always fires exactly once (success or fail) at the + end of this method. + """ + + # if we're given the NOBLOCK flag act as normal and let the EAGAIN get raised + if flags & zmq.NOBLOCK: + try: + msg = super(_Socket, self).send(data, flags, copy, track) + finally: + if not self.__in_send_multipart: + self.__state_changed() + return msg + # ensure the zmq.NOBLOCK flag is part of flags + flags |= zmq.NOBLOCK + while True: # Attempt to complete this operation indefinitely, blocking the current greenlet + try: + # attempt the actual call + msg = super(_Socket, self).send(data, flags, copy, track) + except zmq.ZMQError as e: + # if the raised ZMQError is not EAGAIN, reraise + if e.errno != zmq.EAGAIN: + if not self.__in_send_multipart: + self.__state_changed() + raise + else: + if not self.__in_send_multipart: + self.__state_changed() + return msg + # defer to the event loop until we're notified the socket is writable + self._wait_write() + + def recv(self, flags=0, copy=True, track=False): + """recv, which will only block current greenlet + + state_changed always fires exactly once (success or fail) at the + end of this method. + """ + if flags & zmq.NOBLOCK: + try: + msg = super(_Socket, self).recv(flags, copy, track) + finally: + if not self.__in_recv_multipart: + self.__state_changed() + return msg + + flags |= zmq.NOBLOCK + while True: + try: + msg = super(_Socket, self).recv(flags, copy, track) + except zmq.ZMQError as e: + if e.errno != zmq.EAGAIN: + if not self.__in_recv_multipart: + self.__state_changed() + raise + else: + if not self.__in_recv_multipart: + self.__state_changed() + return msg + self._wait_read() + + def send_multipart(self, *args, **kwargs): + """wrap send_multipart to prevent state_changed on each partial send""" + self.__in_send_multipart = True + try: + msg = super(_Socket, self).send_multipart(*args, **kwargs) + finally: + self.__in_send_multipart = False + self.__state_changed() + return msg + + def recv_multipart(self, *args, **kwargs): + """wrap recv_multipart to prevent state_changed on each partial recv""" + self.__in_recv_multipart = True + try: + msg = super(_Socket, self).recv_multipart(*args, **kwargs) + finally: + self.__in_recv_multipart = False + self.__state_changed() + return msg + + def get(self, opt): + """trigger state_changed on getsockopt(EVENTS)""" + if opt in TIMEOS: + warnings.warn("TIMEO socket options have no effect in zmq.green", UserWarning) + optval = super(_Socket, self).get(opt) + if opt == zmq.EVENTS: + self.__state_changed() + return optval + + def set(self, opt, val): + """set socket option""" + if opt in TIMEOS: + warnings.warn("TIMEO socket options have no effect in zmq.green", UserWarning) + return super(_Socket, self).set(opt, val) + + +class _Context(_original_Context): + """Replacement for :class:`zmq.Context` + + Ensures that the greened Socket above is used in calls to `socket`. + """ + _socket_class = _Socket diff --git a/scripts/external_libs/zmq/green/device.py b/scripts/external_libs/zmq/green/device.py new file mode 100644 index 00000000..4b070237 --- /dev/null +++ b/scripts/external_libs/zmq/green/device.py @@ -0,0 +1,32 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import zmq +from zmq.green import Poller + +def device(device_type, isocket, osocket): + """Start a zeromq device (gevent-compatible). + + Unlike the true zmq.device, this does not release the GIL. + + Parameters + ---------- + device_type : (QUEUE, FORWARDER, STREAMER) + The type of device to start (ignored). + isocket : Socket + The Socket instance for the incoming traffic. + osocket : Socket + The Socket instance for the outbound traffic. + """ + p = Poller() + if osocket == -1: + osocket = isocket + p.register(isocket, zmq.POLLIN) + p.register(osocket, zmq.POLLIN) + + while True: + events = dict(p.poll()) + if isocket in events: + osocket.send_multipart(isocket.recv_multipart()) + if osocket in events: + isocket.send_multipart(osocket.recv_multipart()) diff --git a/scripts/external_libs/zmq/green/eventloop/__init__.py b/scripts/external_libs/zmq/green/eventloop/__init__.py new file mode 100644 index 00000000..c5150efe --- /dev/null +++ b/scripts/external_libs/zmq/green/eventloop/__init__.py @@ -0,0 +1,3 @@ +from zmq.green.eventloop.ioloop import IOLoop + +__all__ = ['IOLoop']
\ No newline at end of file diff --git a/scripts/external_libs/zmq/green/eventloop/ioloop.py b/scripts/external_libs/zmq/green/eventloop/ioloop.py new file mode 100644 index 00000000..e12fd5e9 --- /dev/null +++ b/scripts/external_libs/zmq/green/eventloop/ioloop.py @@ -0,0 +1,33 @@ +from zmq.eventloop.ioloop import * +from zmq.green import Poller + +RealIOLoop = IOLoop +RealZMQPoller = ZMQPoller + +class IOLoop(RealIOLoop): + + def initialize(self, impl=None): + impl = _poll() if impl is None else impl + super(IOLoop, self).initialize(impl) + + @staticmethod + def instance(): + """Returns a global `IOLoop` instance. + + Most applications have a single, global `IOLoop` running on the + main thread. Use this method to get this instance from + another thread. To get the current thread's `IOLoop`, use `current()`. + """ + # install this class as the active IOLoop implementation + # when using tornado 3 + if tornado_version >= (3,): + PollIOLoop.configure(IOLoop) + return PollIOLoop.instance() + + +class ZMQPoller(RealZMQPoller): + """gevent-compatible version of ioloop.ZMQPoller""" + def __init__(self): + self._poller = Poller() + +_poll = ZMQPoller diff --git a/scripts/external_libs/zmq/green/eventloop/zmqstream.py b/scripts/external_libs/zmq/green/eventloop/zmqstream.py new file mode 100644 index 00000000..90fbd1f5 --- /dev/null +++ b/scripts/external_libs/zmq/green/eventloop/zmqstream.py @@ -0,0 +1,11 @@ +from zmq.eventloop.zmqstream import * + +from zmq.green.eventloop.ioloop import IOLoop + +RealZMQStream = ZMQStream + +class ZMQStream(RealZMQStream): + + def __init__(self, socket, io_loop=None): + io_loop = io_loop or IOLoop.instance() + super(ZMQStream, self).__init__(socket, io_loop=io_loop) diff --git a/scripts/external_libs/zmq/green/poll.py b/scripts/external_libs/zmq/green/poll.py new file mode 100644 index 00000000..8f016129 --- /dev/null +++ b/scripts/external_libs/zmq/green/poll.py @@ -0,0 +1,95 @@ +import zmq +import gevent +from gevent import select + +from zmq import Poller as _original_Poller + + +class _Poller(_original_Poller): + """Replacement for :class:`zmq.Poller` + + Ensures that the greened Poller below is used in calls to + :meth:`zmq.Poller.poll`. + """ + _gevent_bug_timeout = 1.33 # minimum poll interval, for working around gevent bug + + def _get_descriptors(self): + """Returns three elements tuple with socket descriptors ready + for gevent.select.select + """ + rlist = [] + wlist = [] + xlist = [] + + for socket, flags in self.sockets: + if isinstance(socket, zmq.Socket): + rlist.append(socket.getsockopt(zmq.FD)) + continue + elif isinstance(socket, int): + fd = socket + elif hasattr(socket, 'fileno'): + try: + fd = int(socket.fileno()) + except: + raise ValueError('fileno() must return an valid integer fd') + else: + raise TypeError('Socket must be a 0MQ socket, an integer fd ' + 'or have a fileno() method: %r' % socket) + + if flags & zmq.POLLIN: + rlist.append(fd) + if flags & zmq.POLLOUT: + wlist.append(fd) + if flags & zmq.POLLERR: + xlist.append(fd) + + return (rlist, wlist, xlist) + + def poll(self, timeout=-1): + """Overridden method to ensure that the green version of + Poller is used. + + Behaves the same as :meth:`zmq.core.Poller.poll` + """ + + if timeout is None: + timeout = -1 + + if timeout < 0: + timeout = -1 + + rlist = None + wlist = None + xlist = None + + if timeout > 0: + tout = gevent.Timeout.start_new(timeout/1000.0) + + try: + # Loop until timeout or events available + rlist, wlist, xlist = self._get_descriptors() + while True: + events = super(_Poller, self).poll(0) + if events or timeout == 0: + return events + + # wait for activity on sockets in a green way + # set a minimum poll frequency, + # because gevent < 1.0 cannot be trusted to catch edge-triggered FD events + _bug_timeout = gevent.Timeout.start_new(self._gevent_bug_timeout) + try: + select.select(rlist, wlist, xlist) + except gevent.Timeout as t: + if t is not _bug_timeout: + raise + finally: + _bug_timeout.cancel() + + except gevent.Timeout as t: + if t is not tout: + raise + return [] + finally: + if timeout > 0: + tout.cancel() + diff --git a/scripts/external_libs/zmq/libzmq.so b/scripts/external_libs/zmq/libzmq.so Binary files differnew file mode 100644 index 00000000..16980c27 --- /dev/null +++ b/scripts/external_libs/zmq/libzmq.so diff --git a/scripts/external_libs/zmq/log/__init__.py b/scripts/external_libs/zmq/log/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/scripts/external_libs/zmq/log/__init__.py diff --git a/scripts/external_libs/zmq/log/handlers.py b/scripts/external_libs/zmq/log/handlers.py new file mode 100644 index 00000000..5ff21bf3 --- /dev/null +++ b/scripts/external_libs/zmq/log/handlers.py @@ -0,0 +1,146 @@ +"""pyzmq logging handlers. + +This mainly defines the PUBHandler object for publishing logging messages over +a zmq.PUB socket. + +The PUBHandler can be used with the regular logging module, as in:: + + >>> import logging + >>> handler = PUBHandler('tcp://127.0.0.1:12345') + >>> handler.root_topic = 'foo' + >>> logger = logging.getLogger('foobar') + >>> logger.setLevel(logging.DEBUG) + >>> logger.addHandler(handler) + +After this point, all messages logged by ``logger`` will be published on the +PUB socket. + +Code adapted from StarCluster: + + http://github.com/jtriley/StarCluster/blob/master/starcluster/logger.py +""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import logging +from logging import INFO, DEBUG, WARN, ERROR, FATAL + +import zmq +from zmq.utils.strtypes import bytes, unicode, cast_bytes + + +TOPIC_DELIM="::" # delimiter for splitting topics on the receiving end. + + +class PUBHandler(logging.Handler): + """A basic logging handler that emits log messages through a PUB socket. + + Takes a PUB socket already bound to interfaces or an interface to bind to. + + Example:: + + sock = context.socket(zmq.PUB) + sock.bind('inproc://log') + handler = PUBHandler(sock) + + Or:: + + handler = PUBHandler('inproc://loc') + + These are equivalent. + + Log messages handled by this handler are broadcast with ZMQ topics + ``this.root_topic`` comes first, followed by the log level + (DEBUG,INFO,etc.), followed by any additional subtopics specified in the + message by: log.debug("subtopic.subsub::the real message") + """ + root_topic="" + socket = None + + formatters = { + logging.DEBUG: logging.Formatter( + "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n"), + logging.INFO: logging.Formatter("%(message)s\n"), + logging.WARN: logging.Formatter( + "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n"), + logging.ERROR: logging.Formatter( + "%(levelname)s %(filename)s:%(lineno)d - %(message)s - %(exc_info)s\n"), + logging.CRITICAL: logging.Formatter( + "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n")} + + def __init__(self, interface_or_socket, context=None): + logging.Handler.__init__(self) + if isinstance(interface_or_socket, zmq.Socket): + self.socket = interface_or_socket + self.ctx = self.socket.context + else: + self.ctx = context or zmq.Context() + self.socket = self.ctx.socket(zmq.PUB) + self.socket.bind(interface_or_socket) + + def format(self,record): + """Format a record.""" + return self.formatters[record.levelno].format(record) + + def emit(self, record): + """Emit a log message on my socket.""" + try: + topic, record.msg = record.msg.split(TOPIC_DELIM,1) + except Exception: + topic = "" + try: + bmsg = cast_bytes(self.format(record)) + except Exception: + self.handleError(record) + return + + topic_list = [] + + if self.root_topic: + topic_list.append(self.root_topic) + + topic_list.append(record.levelname) + + if topic: + topic_list.append(topic) + + btopic = b'.'.join(cast_bytes(t) for t in topic_list) + + self.socket.send_multipart([btopic, bmsg]) + + +class TopicLogger(logging.Logger): + """A simple wrapper that takes an additional argument to log methods. + + All the regular methods exist, but instead of one msg argument, two + arguments: topic, msg are passed. + + That is:: + + logger.debug('msg') + + Would become:: + + logger.debug('topic.sub', 'msg') + """ + def log(self, level, topic, msg, *args, **kwargs): + """Log 'msg % args' with level and topic. + + To pass exception information, use the keyword argument exc_info + with a True value:: + + logger.log(level, "zmq.fun", "We have a %s", + "mysterious problem", exc_info=1) + """ + logging.Logger.log(self, level, '%s::%s'%(topic,msg), *args, **kwargs) + +# Generate the methods of TopicLogger, since they are just adding a +# topic prefix to a message. +for name in "debug warn warning error critical fatal".split(): + meth = getattr(logging.Logger,name) + setattr(TopicLogger, name, + lambda self, level, topic, msg, *args, **kwargs: + meth(self, level, topic+TOPIC_DELIM+msg,*args, **kwargs)) + diff --git a/scripts/external_libs/zmq/ssh/__init__.py b/scripts/external_libs/zmq/ssh/__init__.py new file mode 100644 index 00000000..57f09568 --- /dev/null +++ b/scripts/external_libs/zmq/ssh/__init__.py @@ -0,0 +1 @@ +from zmq.ssh.tunnel import * diff --git a/scripts/external_libs/zmq/ssh/forward.py b/scripts/external_libs/zmq/ssh/forward.py new file mode 100644 index 00000000..2d619462 --- /dev/null +++ b/scripts/external_libs/zmq/ssh/forward.py @@ -0,0 +1,91 @@ +# +# This file is adapted from a paramiko demo, and thus licensed under LGPL 2.1. +# Original Copyright (C) 2003-2007 Robey Pointer <robeypointer@gmail.com> +# Edits Copyright (C) 2010 The IPython Team +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA. + +""" +Sample script showing how to do local port forwarding over paramiko. + +This script connects to the requested SSH server and sets up local port +forwarding (the openssh -L option) from a local port through a tunneled +connection to a destination reachable from the SSH server machine. +""" + +from __future__ import print_function + +import logging +import select +try: # Python 3 + import socketserver +except ImportError: # Python 2 + import SocketServer as socketserver + +logger = logging.getLogger('ssh') + +class ForwardServer (socketserver.ThreadingTCPServer): + daemon_threads = True + allow_reuse_address = True + + +class Handler (socketserver.BaseRequestHandler): + + def handle(self): + try: + chan = self.ssh_transport.open_channel('direct-tcpip', + (self.chain_host, self.chain_port), + self.request.getpeername()) + except Exception as e: + logger.debug('Incoming request to %s:%d failed: %s' % (self.chain_host, + self.chain_port, + repr(e))) + return + if chan is None: + logger.debug('Incoming request to %s:%d was rejected by the SSH server.' % + (self.chain_host, self.chain_port)) + return + + logger.debug('Connected! Tunnel open %r -> %r -> %r' % (self.request.getpeername(), + chan.getpeername(), (self.chain_host, self.chain_port))) + while True: + r, w, x = select.select([self.request, chan], [], []) + if self.request in r: + data = self.request.recv(1024) + if len(data) == 0: + break + chan.send(data) + if chan in r: + data = chan.recv(1024) + if len(data) == 0: + break + self.request.send(data) + chan.close() + self.request.close() + logger.debug('Tunnel closed ') + + +def forward_tunnel(local_port, remote_host, remote_port, transport): + # this is a little convoluted, but lets me configure things for the Handler + # object. (SocketServer doesn't give Handlers any way to access the outer + # server normally.) + class SubHander (Handler): + chain_host = remote_host + chain_port = remote_port + ssh_transport = transport + ForwardServer(('127.0.0.1', local_port), SubHander).serve_forever() + + +__all__ = ['forward_tunnel'] diff --git a/scripts/external_libs/zmq/ssh/tunnel.py b/scripts/external_libs/zmq/ssh/tunnel.py new file mode 100644 index 00000000..5a0c5433 --- /dev/null +++ b/scripts/external_libs/zmq/ssh/tunnel.py @@ -0,0 +1,376 @@ +"""Basic ssh tunnel utilities, and convenience functions for tunneling +zeromq connections. +""" + +# Copyright (C) 2010-2011 IPython Development Team +# Copyright (C) 2011- PyZMQ Developers +# +# Redistributed from IPython under the terms of the BSD License. + + +from __future__ import print_function + +import atexit +import os +import signal +import socket +import sys +import warnings +from getpass import getpass, getuser +from multiprocessing import Process + +try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import paramiko + SSHException = paramiko.ssh_exception.SSHException +except ImportError: + paramiko = None + class SSHException(Exception): + pass +else: + from .forward import forward_tunnel + +try: + import pexpect +except ImportError: + pexpect = None + + +_random_ports = set() + +def select_random_ports(n): + """Selects and return n random ports that are available.""" + ports = [] + for i in range(n): + sock = socket.socket() + sock.bind(('', 0)) + while sock.getsockname()[1] in _random_ports: + sock.close() + sock = socket.socket() + sock.bind(('', 0)) + ports.append(sock) + for i, sock in enumerate(ports): + port = sock.getsockname()[1] + sock.close() + ports[i] = port + _random_ports.add(port) + return ports + + +#----------------------------------------------------------------------------- +# Check for passwordless login +#----------------------------------------------------------------------------- + +def try_passwordless_ssh(server, keyfile, paramiko=None): + """Attempt to make an ssh connection without a password. + This is mainly used for requiring password input only once + when many tunnels may be connected to the same server. + + If paramiko is None, the default for the platform is chosen. + """ + if paramiko is None: + paramiko = sys.platform == 'win32' + if not paramiko: + f = _try_passwordless_openssh + else: + f = _try_passwordless_paramiko + return f(server, keyfile) + +def _try_passwordless_openssh(server, keyfile): + """Try passwordless login with shell ssh command.""" + if pexpect is None: + raise ImportError("pexpect unavailable, use paramiko") + cmd = 'ssh -f '+ server + if keyfile: + cmd += ' -i ' + keyfile + cmd += ' exit' + + # pop SSH_ASKPASS from env + env = os.environ.copy() + env.pop('SSH_ASKPASS', None) + + ssh_newkey = 'Are you sure you want to continue connecting' + p = pexpect.spawn(cmd, env=env) + while True: + try: + i = p.expect([ssh_newkey, '[Pp]assword:'], timeout=.1) + if i==0: + raise SSHException('The authenticity of the host can\'t be established.') + except pexpect.TIMEOUT: + continue + except pexpect.EOF: + return True + else: + return False + +def _try_passwordless_paramiko(server, keyfile): + """Try passwordless login with paramiko.""" + if paramiko is None: + msg = "Paramiko unavaliable, " + if sys.platform == 'win32': + msg += "Paramiko is required for ssh tunneled connections on Windows." + else: + msg += "use OpenSSH." + raise ImportError(msg) + username, server, port = _split_server(server) + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.WarningPolicy()) + try: + client.connect(server, port, username=username, key_filename=keyfile, + look_for_keys=True) + except paramiko.AuthenticationException: + return False + else: + client.close() + return True + + +def tunnel_connection(socket, addr, server, keyfile=None, password=None, paramiko=None, timeout=60): + """Connect a socket to an address via an ssh tunnel. + + This is a wrapper for socket.connect(addr), when addr is not accessible + from the local machine. It simply creates an ssh tunnel using the remaining args, + and calls socket.connect('tcp://localhost:lport') where lport is the randomly + selected local port of the tunnel. + + """ + new_url, tunnel = open_tunnel(addr, server, keyfile=keyfile, password=password, paramiko=paramiko, timeout=timeout) + socket.connect(new_url) + return tunnel + + +def open_tunnel(addr, server, keyfile=None, password=None, paramiko=None, timeout=60): + """Open a tunneled connection from a 0MQ url. + + For use inside tunnel_connection. + + Returns + ------- + + (url, tunnel) : (str, object) + The 0MQ url that has been forwarded, and the tunnel object + """ + + lport = select_random_ports(1)[0] + transport, addr = addr.split('://') + ip,rport = addr.split(':') + rport = int(rport) + if paramiko is None: + paramiko = sys.platform == 'win32' + if paramiko: + tunnelf = paramiko_tunnel + else: + tunnelf = openssh_tunnel + + tunnel = tunnelf(lport, rport, server, remoteip=ip, keyfile=keyfile, password=password, timeout=timeout) + return 'tcp://127.0.0.1:%i'%lport, tunnel + +def openssh_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=60): + """Create an ssh tunnel using command-line ssh that connects port lport + on this machine to localhost:rport on server. The tunnel + will automatically close when not in use, remaining open + for a minimum of timeout seconds for an initial connection. + + This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`, + as seen from `server`. + + keyfile and password may be specified, but ssh config is checked for defaults. + + Parameters + ---------- + + lport : int + local port for connecting to the tunnel from this machine. + rport : int + port on the remote machine to connect to. + server : str + The ssh server to connect to. The full ssh server string will be parsed. + user@server:port + remoteip : str [Default: 127.0.0.1] + The remote ip, specifying the destination of the tunnel. + Default is localhost, which means that the tunnel would redirect + localhost:lport on this machine to localhost:rport on the *server*. + + keyfile : str; path to public key file + This specifies a key to be used in ssh login, default None. + Regular default ssh keys will be used without specifying this argument. + password : str; + Your ssh password to the ssh server. Note that if this is left None, + you will be prompted for it if passwordless key based login is unavailable. + timeout : int [default: 60] + The time (in seconds) after which no activity will result in the tunnel + closing. This prevents orphaned tunnels from running forever. + """ + if pexpect is None: + raise ImportError("pexpect unavailable, use paramiko_tunnel") + ssh="ssh " + if keyfile: + ssh += "-i " + keyfile + + if ':' in server: + server, port = server.split(':') + ssh += " -p %s" % port + + cmd = "%s -O check %s" % (ssh, server) + (output, exitstatus) = pexpect.run(cmd, withexitstatus=True) + if not exitstatus: + pid = int(output[output.find("(pid=")+5:output.find(")")]) + cmd = "%s -O forward -L 127.0.0.1:%i:%s:%i %s" % ( + ssh, lport, remoteip, rport, server) + (output, exitstatus) = pexpect.run(cmd, withexitstatus=True) + if not exitstatus: + atexit.register(_stop_tunnel, cmd.replace("-O forward", "-O cancel", 1)) + return pid + cmd = "%s -f -S none -L 127.0.0.1:%i:%s:%i %s sleep %i" % ( + ssh, lport, remoteip, rport, server, timeout) + + # pop SSH_ASKPASS from env + env = os.environ.copy() + env.pop('SSH_ASKPASS', None) + + ssh_newkey = 'Are you sure you want to continue connecting' + tunnel = pexpect.spawn(cmd, env=env) + failed = False + while True: + try: + i = tunnel.expect([ssh_newkey, '[Pp]assword:'], timeout=.1) + if i==0: + raise SSHException('The authenticity of the host can\'t be established.') + except pexpect.TIMEOUT: + continue + except pexpect.EOF: + if tunnel.exitstatus: + print(tunnel.exitstatus) + print(tunnel.before) + print(tunnel.after) + raise RuntimeError("tunnel '%s' failed to start"%(cmd)) + else: + return tunnel.pid + else: + if failed: + print("Password rejected, try again") + password=None + if password is None: + password = getpass("%s's password: "%(server)) + tunnel.sendline(password) + failed = True + +def _stop_tunnel(cmd): + pexpect.run(cmd) + +def _split_server(server): + if '@' in server: + username,server = server.split('@', 1) + else: + username = getuser() + if ':' in server: + server, port = server.split(':') + port = int(port) + else: + port = 22 + return username, server, port + +def paramiko_tunnel(lport, rport, server, remoteip='127.0.0.1', keyfile=None, password=None, timeout=60): + """launch a tunner with paramiko in a subprocess. This should only be used + when shell ssh is unavailable (e.g. Windows). + + This creates a tunnel redirecting `localhost:lport` to `remoteip:rport`, + as seen from `server`. + + If you are familiar with ssh tunnels, this creates the tunnel: + + ssh server -L localhost:lport:remoteip:rport + + keyfile and password may be specified, but ssh config is checked for defaults. + + + Parameters + ---------- + + lport : int + local port for connecting to the tunnel from this machine. + rport : int + port on the remote machine to connect to. + server : str + The ssh server to connect to. The full ssh server string will be parsed. + user@server:port + remoteip : str [Default: 127.0.0.1] + The remote ip, specifying the destination of the tunnel. + Default is localhost, which means that the tunnel would redirect + localhost:lport on this machine to localhost:rport on the *server*. + + keyfile : str; path to public key file + This specifies a key to be used in ssh login, default None. + Regular default ssh keys will be used without specifying this argument. + password : str; + Your ssh password to the ssh server. Note that if this is left None, + you will be prompted for it if passwordless key based login is unavailable. + timeout : int [default: 60] + The time (in seconds) after which no activity will result in the tunnel + closing. This prevents orphaned tunnels from running forever. + + """ + if paramiko is None: + raise ImportError("Paramiko not available") + + if password is None: + if not _try_passwordless_paramiko(server, keyfile): + password = getpass("%s's password: "%(server)) + + p = Process(target=_paramiko_tunnel, + args=(lport, rport, server, remoteip), + kwargs=dict(keyfile=keyfile, password=password)) + p.daemon=False + p.start() + atexit.register(_shutdown_process, p) + return p + +def _shutdown_process(p): + if p.is_alive(): + p.terminate() + +def _paramiko_tunnel(lport, rport, server, remoteip, keyfile=None, password=None): + """Function for actually starting a paramiko tunnel, to be passed + to multiprocessing.Process(target=this), and not called directly. + """ + username, server, port = _split_server(server) + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.WarningPolicy()) + + try: + client.connect(server, port, username=username, key_filename=keyfile, + look_for_keys=True, password=password) +# except paramiko.AuthenticationException: +# if password is None: +# password = getpass("%s@%s's password: "%(username, server)) +# client.connect(server, port, username=username, password=password) +# else: +# raise + except Exception as e: + print('*** Failed to connect to %s:%d: %r' % (server, port, e)) + sys.exit(1) + + # Don't let SIGINT kill the tunnel subprocess + signal.signal(signal.SIGINT, signal.SIG_IGN) + + try: + forward_tunnel(lport, remoteip, rport, client.get_transport()) + except KeyboardInterrupt: + print('SIGINT: Port forwarding stopped cleanly') + sys.exit(0) + except Exception as e: + print("Port forwarding stopped uncleanly: %s"%e) + sys.exit(255) + +if sys.platform == 'win32': + ssh_tunnel = paramiko_tunnel +else: + ssh_tunnel = openssh_tunnel + + +__all__ = ['tunnel_connection', 'ssh_tunnel', 'openssh_tunnel', 'paramiko_tunnel', 'try_passwordless_ssh'] + + diff --git a/scripts/external_libs/zmq/sugar/__init__.py b/scripts/external_libs/zmq/sugar/__init__.py new file mode 100644 index 00000000..d0510a44 --- /dev/null +++ b/scripts/external_libs/zmq/sugar/__init__.py @@ -0,0 +1,27 @@ +"""pure-Python sugar wrappers for core 0MQ objects.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +from zmq.sugar import ( + constants, context, frame, poll, socket, tracker, version +) +from zmq import error + +__all__ = ['constants'] +for submod in ( + constants, context, error, frame, poll, socket, tracker, version +): + __all__.extend(submod.__all__) + +from zmq.error import * +from zmq.sugar.context import * +from zmq.sugar.tracker import * +from zmq.sugar.socket import * +from zmq.sugar.constants import * +from zmq.sugar.frame import * +from zmq.sugar.poll import * +# from zmq.sugar.stopwatch import * +# from zmq.sugar._device import * +from zmq.sugar.version import * diff --git a/scripts/external_libs/zmq/sugar/attrsettr.py b/scripts/external_libs/zmq/sugar/attrsettr.py new file mode 100644 index 00000000..4bbd36d6 --- /dev/null +++ b/scripts/external_libs/zmq/sugar/attrsettr.py @@ -0,0 +1,52 @@ +# coding: utf-8 +"""Mixin for mapping set/getattr to self.set/get""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +from . import constants + +class AttributeSetter(object): + + def __setattr__(self, key, value): + """set zmq options by attribute""" + + # regular setattr only allowed for class-defined attributes + for obj in [self] + self.__class__.mro(): + if key in obj.__dict__: + object.__setattr__(self, key, value) + return + + upper_key = key.upper() + try: + opt = getattr(constants, upper_key) + except AttributeError: + raise AttributeError("%s has no such option: %s" % ( + self.__class__.__name__, upper_key) + ) + else: + self._set_attr_opt(upper_key, opt, value) + + def _set_attr_opt(self, name, opt, value): + """override if setattr should do something other than call self.set""" + self.set(opt, value) + + def __getattr__(self, key): + """get zmq options by attribute""" + upper_key = key.upper() + try: + opt = getattr(constants, upper_key) + except AttributeError: + raise AttributeError("%s has no such option: %s" % ( + self.__class__.__name__, upper_key) + ) + else: + return self._get_attr_opt(upper_key, opt) + + def _get_attr_opt(self, name, opt): + """override if getattr should do something other than call self.get""" + return self.get(opt) + + +__all__ = ['AttributeSetter'] diff --git a/scripts/external_libs/zmq/sugar/constants.py b/scripts/external_libs/zmq/sugar/constants.py new file mode 100644 index 00000000..88281176 --- /dev/null +++ b/scripts/external_libs/zmq/sugar/constants.py @@ -0,0 +1,98 @@ +"""0MQ Constants.""" + +# Copyright (c) PyZMQ Developers. +# Distributed under the terms of the Modified BSD License. + +from zmq.backend import constants +from zmq.utils.constant_names import ( + base_names, + switched_sockopt_names, + int_sockopt_names, + int64_sockopt_names, + bytes_sockopt_names, + fd_sockopt_names, + ctx_opt_names, + msg_opt_names, +) + +#----------------------------------------------------------------------------- +# Python module level constants +#----------------------------------------------------------------------------- + +__all__ = [ + 'int_sockopts', + 'int64_sockopts', + 'bytes_sockopts', + 'ctx_opts', + 'ctx_opt_names', + ] + +int_sockopts = set() +int64_sockopts = set() +bytes_sockopts = set() +fd_sockopts = set() +ctx_opts = set() +msg_opts = set() + + +if constants.VERSION < 30000: + int64_sockopt_names.extend(switched_sockopt_names) +else: + int_sockopt_names.extend(switched_sockopt_names) + +_UNDEFINED = -9999 + +def _add_constant(name, container=None): + """add a constant to be defined + + optionally add it to one of the sets for use in get/setopt checkers + """ + c = getattr(constants, name, _UNDEFINED) + if c == _UNDEFINED: + return + globals()[name] = c + __all__.append(name) + if container is not None: + container.add(c) + return c + +for name in base_names: + _add_constant(name) + +for name in int_sockopt_names: + _add_constant(name, int_sockopts) + +for name in int64_sockopt_names: + _add_constant(name, int64_sockopts) + +for name in bytes_sockopt_names: + _add_constant(name, bytes_sockopts) + +for name in fd_sockopt_names: + _add_constant(name, fd_sockopts) + +for name in ctx_opt_names: + _add_constant(name, ctx_opts) + +for name in msg_opt_names: + _add_constant(name, msg_opts) + +# ensure some aliases are always defined +aliases = [ + ('DONTWAIT', 'NOBLOCK'), + ('XREQ', 'DEALER'), + ('XREP', 'ROUTER'), +] +for group in aliases: + undefined = set() + found = None + for name in group: + value = getattr(constants, name, -1) + if value != -1: + found = value + else: + undefined.add(name) + if found is not None: + for name in undefined: + globals()[name] = found + __all__.append(name) diff --git a/scripts/external_libs/zmq/sugar/context.py b/scripts/external_libs/zmq/sugar/context.py new file mode 100644 index 00000000..86a9c5dc --- /dev/null +++ b/scripts/external_libs/zmq/sugar/context.py @@ -0,0 +1,192 @@ +# coding: utf-8 +"""Python bindings for 0MQ.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import atexit +import weakref + +from zmq.backend import Context as ContextBase +from . import constants +from .attrsettr import AttributeSetter +from .constants import ENOTSUP, ctx_opt_names +from .socket import Socket +from zmq.error import ZMQError + +from zmq.utils.interop import cast_int_addr + + +class Context(ContextBase, AttributeSetter): + """Create a zmq Context + + A zmq Context creates sockets via its ``ctx.socket`` method. + """ + sockopts = None + _instance = None + _shadow = False + _exiting = False + + def __init__(self, io_threads=1, **kwargs): + super(Context, self).__init__(io_threads=io_threads, **kwargs) + if kwargs.get('shadow', False): + self._shadow = True + else: + self._shadow = False + self.sockopts = {} + + self._exiting = False + if not self._shadow: + ctx_ref = weakref.ref(self) + def _notify_atexit(): + ctx = ctx_ref() + if ctx is not None: + ctx._exiting = True + atexit.register(_notify_atexit) + + def __del__(self): + """deleting a Context should terminate it, without trying non-threadsafe destroy""" + if not self._shadow and not self._exiting: + self.term() + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.term() + + @classmethod + def shadow(cls, address): + """Shadow an existing libzmq context + + address is the integer address of the libzmq context + or an FFI pointer to it. + + .. versionadded:: 14.1 + """ + address = cast_int_addr(address) + return cls(shadow=address) + + @classmethod + def shadow_pyczmq(cls, ctx): + """Shadow an existing pyczmq context + + ctx is the FFI `zctx_t *` pointer + + .. versionadded:: 14.1 + """ + from pyczmq import zctx + + underlying = zctx.underlying(ctx) + address = cast_int_addr(underlying) + return cls(shadow=address) + + # static method copied from tornado IOLoop.instance + @classmethod + def instance(cls, io_threads=1): + """Returns a global Context instance. + + Most single-threaded applications have a single, global Context. + Use this method instead of passing around Context instances + throughout your code. + + A common pattern for classes that depend on Contexts is to use + a default argument to enable programs with multiple Contexts + but not require the argument for simpler applications: + + class MyClass(object): + def __init__(self, context=None): + self.context = context or Context.instance() + """ + if cls._instance is None or cls._instance.closed: + cls._instance = cls(io_threads=io_threads) + return cls._instance + + #------------------------------------------------------------------------- + # Hooks for ctxopt completion + #------------------------------------------------------------------------- + + def __dir__(self): + keys = dir(self.__class__) + + for collection in ( + ctx_opt_names, + ): + keys.extend(collection) + return keys + + #------------------------------------------------------------------------- + # Creating Sockets + #------------------------------------------------------------------------- + + @property + def _socket_class(self): + return Socket + + def socket(self, socket_type): + """Create a Socket associated with this Context. + + Parameters + ---------- + socket_type : int + The socket type, which can be any of the 0MQ socket types: + REQ, REP, PUB, SUB, PAIR, DEALER, ROUTER, PULL, PUSH, etc. + """ + if self.closed: + raise ZMQError(ENOTSUP) + s = self._socket_class(self, socket_type) + for opt, value in self.sockopts.items(): + try: + s.setsockopt(opt, value) + except ZMQError: + # ignore ZMQErrors, which are likely for socket options + # that do not apply to a particular socket type, e.g. + # SUBSCRIBE for non-SUB sockets. + pass + return s + + def setsockopt(self, opt, value): + """set default socket options for new sockets created by this Context + + .. versionadded:: 13.0 + """ + self.sockopts[opt] = value + + def getsockopt(self, opt): + """get default socket options for new sockets created by this Context + + .. versionadded:: 13.0 + """ + return self.sockopts[opt] + + def _set_attr_opt(self, name, opt, value): + """set default sockopts as attributes""" + if name in constants.ctx_opt_names: + return self.set(opt, value) + else: + self.sockopts[opt] = value + + def _get_attr_opt(self, name, opt): + """get default sockopts as attributes""" + if name in constants.ctx_opt_names: + return self.get(opt) + else: + if opt not in self.sockopts: + raise AttributeError(name) + else: + return self.sockopts[opt] + + def __delattr__(self, key): + """delete default sockopts as attributes""" + key = key.upper() + try: + opt = getattr(constants, key) + except AttributeError: + raise AttributeError("no such socket option: %s" % key) + else: + if opt not in self.sockopts: + raise AttributeError(key) + else: + del self.sockopts[opt] + +__all__ = ['Context'] diff --git a/scripts/external_libs/zmq/sugar/frame.py b/scripts/external_libs/zmq/sugar/frame.py new file mode 100644 index 00000000..9f556c86 --- /dev/null +++ b/scripts/external_libs/zmq/sugar/frame.py @@ -0,0 +1,19 @@ +# coding: utf-8 +"""0MQ Frame pure Python methods.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +from .attrsettr import AttributeSetter +from zmq.backend import Frame as FrameBase + + +class Frame(FrameBase, AttributeSetter): + def __getitem__(self, key): + # map Frame['User-Id'] to Frame.get('User-Id') + return self.get(key) + +# keep deprecated alias +Message = Frame +__all__ = ['Frame', 'Message']
\ No newline at end of file diff --git a/scripts/external_libs/zmq/sugar/poll.py b/scripts/external_libs/zmq/sugar/poll.py new file mode 100644 index 00000000..c7b1d1bb --- /dev/null +++ b/scripts/external_libs/zmq/sugar/poll.py @@ -0,0 +1,161 @@ +"""0MQ polling related functions and classes.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import zmq +from zmq.backend import zmq_poll +from .constants import POLLIN, POLLOUT, POLLERR + +#----------------------------------------------------------------------------- +# Polling related methods +#----------------------------------------------------------------------------- + + +class Poller(object): + """A stateful poll interface that mirrors Python's built-in poll.""" + sockets = None + _map = {} + + def __init__(self): + self.sockets = [] + self._map = {} + + def __contains__(self, socket): + return socket in self._map + + def register(self, socket, flags=POLLIN|POLLOUT): + """p.register(socket, flags=POLLIN|POLLOUT) + + Register a 0MQ socket or native fd for I/O monitoring. + + register(s,0) is equivalent to unregister(s). + + Parameters + ---------- + socket : zmq.Socket or native socket + A zmq.Socket or any Python object having a ``fileno()`` + method that returns a valid file descriptor. + flags : int + The events to watch for. Can be POLLIN, POLLOUT or POLLIN|POLLOUT. + If `flags=0`, socket will be unregistered. + """ + if flags: + if socket in self._map: + idx = self._map[socket] + self.sockets[idx] = (socket, flags) + else: + idx = len(self.sockets) + self.sockets.append((socket, flags)) + self._map[socket] = idx + elif socket in self._map: + # uregister sockets registered with no events + self.unregister(socket) + else: + # ignore new sockets with no events + pass + + def modify(self, socket, flags=POLLIN|POLLOUT): + """Modify the flags for an already registered 0MQ socket or native fd.""" + self.register(socket, flags) + + def unregister(self, socket): + """Remove a 0MQ socket or native fd for I/O monitoring. + + Parameters + ---------- + socket : Socket + The socket instance to stop polling. + """ + idx = self._map.pop(socket) + self.sockets.pop(idx) + # shift indices after deletion + for socket, flags in self.sockets[idx:]: + self._map[socket] -= 1 + + def poll(self, timeout=None): + """Poll the registered 0MQ or native fds for I/O. + + Parameters + ---------- + timeout : float, int + The timeout in milliseconds. If None, no `timeout` (infinite). This + is in milliseconds to be compatible with ``select.poll()``. The + underlying zmq_poll uses microseconds and we convert to that in + this function. + + Returns + ------- + events : list of tuples + The list of events that are ready to be processed. + This is a list of tuples of the form ``(socket, event)``, where the 0MQ Socket + or integer fd is the first element, and the poll event mask (POLLIN, POLLOUT) is the second. + It is common to call ``events = dict(poller.poll())``, + which turns the list of tuples into a mapping of ``socket : event``. + """ + if timeout is None or timeout < 0: + timeout = -1 + elif isinstance(timeout, float): + timeout = int(timeout) + return zmq_poll(self.sockets, timeout=timeout) + + +def select(rlist, wlist, xlist, timeout=None): + """select(rlist, wlist, xlist, timeout=None) -> (rlist, wlist, xlist) + + Return the result of poll as a lists of sockets ready for r/w/exception. + + This has the same interface as Python's built-in ``select.select()`` function. + + Parameters + ---------- + timeout : float, int, optional + The timeout in seconds. If None, no timeout (infinite). This is in seconds to be + compatible with ``select.select()``. The underlying zmq_poll uses microseconds + and we convert to that in this function. + rlist : list of sockets/FDs + sockets/FDs to be polled for read events + wlist : list of sockets/FDs + sockets/FDs to be polled for write events + xlist : list of sockets/FDs + sockets/FDs to be polled for error events + + Returns + ------- + (rlist, wlist, xlist) : tuple of lists of sockets (length 3) + Lists correspond to sockets available for read/write/error events respectively. + """ + if timeout is None: + timeout = -1 + # Convert from sec -> us for zmq_poll. + # zmq_poll accepts 3.x style timeout in ms + timeout = int(timeout*1000.0) + if timeout < 0: + timeout = -1 + sockets = [] + for s in set(rlist + wlist + xlist): + flags = 0 + if s in rlist: + flags |= POLLIN + if s in wlist: + flags |= POLLOUT + if s in xlist: + flags |= POLLERR + sockets.append((s, flags)) + return_sockets = zmq_poll(sockets, timeout) + rlist, wlist, xlist = [], [], [] + for s, flags in return_sockets: + if flags & POLLIN: + rlist.append(s) + if flags & POLLOUT: + wlist.append(s) + if flags & POLLERR: + xlist.append(s) + return rlist, wlist, xlist + +#----------------------------------------------------------------------------- +# Symbols to export +#----------------------------------------------------------------------------- + +__all__ = [ 'Poller', 'select' ] diff --git a/scripts/external_libs/zmq/sugar/socket.py b/scripts/external_libs/zmq/sugar/socket.py new file mode 100644 index 00000000..c91589d7 --- /dev/null +++ b/scripts/external_libs/zmq/sugar/socket.py @@ -0,0 +1,495 @@ +# coding: utf-8 +"""0MQ Socket pure Python methods.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import codecs +import random +import warnings + +import zmq +from zmq.backend import Socket as SocketBase +from .poll import Poller +from . import constants +from .attrsettr import AttributeSetter +from zmq.error import ZMQError, ZMQBindError +from zmq.utils import jsonapi +from zmq.utils.strtypes import bytes,unicode,basestring +from zmq.utils.interop import cast_int_addr + +from .constants import ( + SNDMORE, ENOTSUP, POLLIN, + int64_sockopt_names, + int_sockopt_names, + bytes_sockopt_names, + fd_sockopt_names, +) +try: + import cPickle + pickle = cPickle +except: + cPickle = None + import pickle + +try: + DEFAULT_PROTOCOL = pickle.DEFAULT_PROTOCOL +except AttributeError: + DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL + + +class Socket(SocketBase, AttributeSetter): + """The ZMQ socket object + + To create a Socket, first create a Context:: + + ctx = zmq.Context.instance() + + then call ``ctx.socket(socket_type)``:: + + s = ctx.socket(zmq.ROUTER) + + """ + _shadow = False + + def __del__(self): + if not self._shadow: + self.close() + + # socket as context manager: + def __enter__(self): + """Sockets are context managers + + .. versionadded:: 14.4 + """ + return self + + def __exit__(self, *args, **kwargs): + self.close() + + #------------------------------------------------------------------------- + # Socket creation + #------------------------------------------------------------------------- + + @classmethod + def shadow(cls, address): + """Shadow an existing libzmq socket + + address is the integer address of the libzmq socket + or an FFI pointer to it. + + .. versionadded:: 14.1 + """ + address = cast_int_addr(address) + return cls(shadow=address) + + #------------------------------------------------------------------------- + # Deprecated aliases + #------------------------------------------------------------------------- + + @property + def socket_type(self): + warnings.warn("Socket.socket_type is deprecated, use Socket.type", + DeprecationWarning + ) + return self.type + + #------------------------------------------------------------------------- + # Hooks for sockopt completion + #------------------------------------------------------------------------- + + def __dir__(self): + keys = dir(self.__class__) + for collection in ( + bytes_sockopt_names, + int_sockopt_names, + int64_sockopt_names, + fd_sockopt_names, + ): + keys.extend(collection) + return keys + + #------------------------------------------------------------------------- + # Getting/Setting options + #------------------------------------------------------------------------- + setsockopt = SocketBase.set + getsockopt = SocketBase.get + + def set_string(self, option, optval, encoding='utf-8'): + """set socket options with a unicode object + + This is simply a wrapper for setsockopt to protect from encoding ambiguity. + + See the 0MQ documentation for details on specific options. + + Parameters + ---------- + option : int + The name of the option to set. Can be any of: SUBSCRIBE, + UNSUBSCRIBE, IDENTITY + optval : unicode string (unicode on py2, str on py3) + The value of the option to set. + encoding : str + The encoding to be used, default is utf8 + """ + if not isinstance(optval, unicode): + raise TypeError("unicode strings only") + return self.set(option, optval.encode(encoding)) + + setsockopt_unicode = setsockopt_string = set_string + + def get_string(self, option, encoding='utf-8'): + """get the value of a socket option + + See the 0MQ documentation for details on specific options. + + Parameters + ---------- + option : int + The option to retrieve. + + Returns + ------- + optval : unicode string (unicode on py2, str on py3) + The value of the option as a unicode string. + """ + + if option not in constants.bytes_sockopts: + raise TypeError("option %i will not return a string to be decoded"%option) + return self.getsockopt(option).decode(encoding) + + getsockopt_unicode = getsockopt_string = get_string + + def bind_to_random_port(self, addr, min_port=49152, max_port=65536, max_tries=100): + """bind this socket to a random port in a range + + Parameters + ---------- + addr : str + The address string without the port to pass to ``Socket.bind()``. + min_port : int, optional + The minimum port in the range of ports to try (inclusive). + max_port : int, optional + The maximum port in the range of ports to try (exclusive). + max_tries : int, optional + The maximum number of bind attempts to make. + + Returns + ------- + port : int + The port the socket was bound to. + + Raises + ------ + ZMQBindError + if `max_tries` reached before successful bind + """ + for i in range(max_tries): + try: + port = random.randrange(min_port, max_port) + self.bind('%s:%s' % (addr, port)) + except ZMQError as exception: + if not exception.errno == zmq.EADDRINUSE: + raise + else: + return port + raise ZMQBindError("Could not bind socket to random port.") + + def get_hwm(self): + """get the High Water Mark + + On libzmq ≥ 3, this gets SNDHWM if available, otherwise RCVHWM + """ + major = zmq.zmq_version_info()[0] + if major >= 3: + # return sndhwm, fallback on rcvhwm + try: + return self.getsockopt(zmq.SNDHWM) + except zmq.ZMQError as e: + pass + + return self.getsockopt(zmq.RCVHWM) + else: + return self.getsockopt(zmq.HWM) + + def set_hwm(self, value): + """set the High Water Mark + + On libzmq ≥ 3, this sets both SNDHWM and RCVHWM + """ + major = zmq.zmq_version_info()[0] + if major >= 3: + raised = None + try: + self.sndhwm = value + except Exception as e: + raised = e + try: + self.rcvhwm = value + except Exception: + raised = e + + if raised: + raise raised + else: + return self.setsockopt(zmq.HWM, value) + + hwm = property(get_hwm, set_hwm, + """property for High Water Mark + + Setting hwm sets both SNDHWM and RCVHWM as appropriate. + It gets SNDHWM if available, otherwise RCVHWM. + """ + ) + + #------------------------------------------------------------------------- + # Sending and receiving messages + #------------------------------------------------------------------------- + + def send_multipart(self, msg_parts, flags=0, copy=True, track=False): + """send a sequence of buffers as a multipart message + + The zmq.SNDMORE flag is added to all msg parts before the last. + + Parameters + ---------- + msg_parts : iterable + A sequence of objects to send as a multipart message. Each element + can be any sendable object (Frame, bytes, buffer-providers) + flags : int, optional + SNDMORE is handled automatically for frames before the last. + copy : bool, optional + Should the frame(s) be sent in a copying or non-copying manner. + track : bool, optional + Should the frame(s) be tracked for notification that ZMQ has + finished with it (ignored if copy=True). + + Returns + ------- + None : if copy or not track + MessageTracker : if track and not copy + a MessageTracker object, whose `pending` property will + be True until the last send is completed. + """ + for msg in msg_parts[:-1]: + self.send(msg, SNDMORE|flags, copy=copy, track=track) + # Send the last part without the extra SNDMORE flag. + return self.send(msg_parts[-1], flags, copy=copy, track=track) + + def recv_multipart(self, flags=0, copy=True, track=False): + """receive a multipart message as a list of bytes or Frame objects + + Parameters + ---------- + flags : int, optional + Any supported flag: NOBLOCK. If NOBLOCK is set, this method + will raise a ZMQError with EAGAIN if a message is not ready. + If NOBLOCK is not set, then this method will block until a + message arrives. + copy : bool, optional + Should the message frame(s) be received in a copying or non-copying manner? + If False a Frame object is returned for each part, if True a copy of + the bytes is made for each frame. + track : bool, optional + Should the message frame(s) be tracked for notification that ZMQ has + finished with it? (ignored if copy=True) + + Returns + ------- + msg_parts : list + A list of frames in the multipart message; either Frames or bytes, + depending on `copy`. + + """ + parts = [self.recv(flags, copy=copy, track=track)] + # have first part already, only loop while more to receive + while self.getsockopt(zmq.RCVMORE): + part = self.recv(flags, copy=copy, track=track) + parts.append(part) + + return parts + + def send_string(self, u, flags=0, copy=True, encoding='utf-8'): + """send a Python unicode string as a message with an encoding + + 0MQ communicates with raw bytes, so you must encode/decode + text (unicode on py2, str on py3) around 0MQ. + + Parameters + ---------- + u : Python unicode string (unicode on py2, str on py3) + The unicode string to send. + flags : int, optional + Any valid send flag. + encoding : str [default: 'utf-8'] + The encoding to be used + """ + if not isinstance(u, basestring): + raise TypeError("unicode/str objects only") + return self.send(u.encode(encoding), flags=flags, copy=copy) + + send_unicode = send_string + + def recv_string(self, flags=0, encoding='utf-8'): + """receive a unicode string, as sent by send_string + + Parameters + ---------- + flags : int + Any valid recv flag. + encoding : str [default: 'utf-8'] + The encoding to be used + + Returns + ------- + s : unicode string (unicode on py2, str on py3) + The Python unicode string that arrives as encoded bytes. + """ + b = self.recv(flags=flags) + return b.decode(encoding) + + recv_unicode = recv_string + + def send_pyobj(self, obj, flags=0, protocol=DEFAULT_PROTOCOL): + """send a Python object as a message using pickle to serialize + + Parameters + ---------- + obj : Python object + The Python object to send. + flags : int + Any valid send flag. + protocol : int + The pickle protocol number to use. The default is pickle.DEFAULT_PROTOCOl + where defined, and pickle.HIGHEST_PROTOCOL elsewhere. + """ + msg = pickle.dumps(obj, protocol) + return self.send(msg, flags) + + def recv_pyobj(self, flags=0): + """receive a Python object as a message using pickle to serialize + + Parameters + ---------- + flags : int + Any valid recv flag. + + Returns + ------- + obj : Python object + The Python object that arrives as a message. + """ + s = self.recv(flags) + return pickle.loads(s) + + def send_json(self, obj, flags=0, **kwargs): + """send a Python object as a message using json to serialize + + Keyword arguments are passed on to json.dumps + + Parameters + ---------- + obj : Python object + The Python object to send + flags : int + Any valid send flag + """ + msg = jsonapi.dumps(obj, **kwargs) + return self.send(msg, flags) + + def recv_json(self, flags=0, **kwargs): + """receive a Python object as a message using json to serialize + + Keyword arguments are passed on to json.loads + + Parameters + ---------- + flags : int + Any valid recv flag. + + Returns + ------- + obj : Python object + The Python object that arrives as a message. + """ + msg = self.recv(flags) + return jsonapi.loads(msg, **kwargs) + + _poller_class = Poller + + def poll(self, timeout=None, flags=POLLIN): + """poll the socket for events + + The default is to poll forever for incoming + events. Timeout is in milliseconds, if specified. + + Parameters + ---------- + timeout : int [default: None] + The timeout (in milliseconds) to wait for an event. If unspecified + (or specified None), will wait forever for an event. + flags : bitfield (int) [default: POLLIN] + The event flags to poll for (any combination of POLLIN|POLLOUT). + The default is to check for incoming events (POLLIN). + + Returns + ------- + events : bitfield (int) + The events that are ready and waiting. Will be 0 if no events were ready + by the time timeout was reached. + """ + + if self.closed: + raise ZMQError(ENOTSUP) + + p = self._poller_class() + p.register(self, flags) + evts = dict(p.poll(timeout)) + # return 0 if no events, otherwise return event bitfield + return evts.get(self, 0) + + def get_monitor_socket(self, events=None, addr=None): + """Return a connected PAIR socket ready to receive the event notifications. + + .. versionadded:: libzmq-4.0 + .. versionadded:: 14.0 + + Parameters + ---------- + events : bitfield (int) [default: ZMQ_EVENTS_ALL] + The bitmask defining which events are wanted. + addr : string [default: None] + The optional endpoint for the monitoring sockets. + + Returns + ------- + socket : (PAIR) + The socket is already connected and ready to receive messages. + """ + # safe-guard, method only available on libzmq >= 4 + if zmq.zmq_version_info() < (4,): + raise NotImplementedError("get_monitor_socket requires libzmq >= 4, have %s" % zmq.zmq_version()) + if addr is None: + # create endpoint name from internal fd + addr = "inproc://monitor.s-%d" % self.FD + if events is None: + # use all events + events = zmq.EVENT_ALL + # attach monitoring socket + self.monitor(addr, events) + # create new PAIR socket and connect it + ret = self.context.socket(zmq.PAIR) + ret.connect(addr) + return ret + + def disable_monitor(self): + """Shutdown the PAIR socket (created using get_monitor_socket) + that is serving socket events. + + .. versionadded:: 14.4 + """ + self.monitor(None, 0) + + +__all__ = ['Socket'] diff --git a/scripts/external_libs/zmq/sugar/tracker.py b/scripts/external_libs/zmq/sugar/tracker.py new file mode 100644 index 00000000..fb8c007f --- /dev/null +++ b/scripts/external_libs/zmq/sugar/tracker.py @@ -0,0 +1,120 @@ +"""Tracker for zero-copy messages with 0MQ.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import time + +try: + # below 3.3 + from threading import _Event as Event +except (ImportError, AttributeError): + # python throws ImportError, cython throws AttributeError + from threading import Event + +from zmq.error import NotDone +from zmq.backend import Frame + +class MessageTracker(object): + """MessageTracker(*towatch) + + A class for tracking if 0MQ is done using one or more messages. + + When you send a 0MQ message, it is not sent immediately. The 0MQ IO thread + sends the message at some later time. Often you want to know when 0MQ has + actually sent the message though. This is complicated by the fact that + a single 0MQ message can be sent multiple times using different sockets. + This class allows you to track all of the 0MQ usages of a message. + + Parameters + ---------- + *towatch : tuple of Event, MessageTracker, Message instances. + This list of objects to track. This class can track the low-level + Events used by the Message class, other MessageTrackers or + actual Messages. + """ + events = None + peers = None + + def __init__(self, *towatch): + """MessageTracker(*towatch) + + Create a message tracker to track a set of mesages. + + Parameters + ---------- + *towatch : tuple of Event, MessageTracker, Message instances. + This list of objects to track. This class can track the low-level + Events used by the Message class, other MessageTrackers or + actual Messages. + """ + self.events = set() + self.peers = set() + for obj in towatch: + if isinstance(obj, Event): + self.events.add(obj) + elif isinstance(obj, MessageTracker): + self.peers.add(obj) + elif isinstance(obj, Frame): + if not obj.tracker: + raise ValueError("Not a tracked message") + self.peers.add(obj.tracker) + else: + raise TypeError("Require Events or Message Frames, not %s"%type(obj)) + + @property + def done(self): + """Is 0MQ completely done with the message(s) being tracked?""" + for evt in self.events: + if not evt.is_set(): + return False + for pm in self.peers: + if not pm.done: + return False + return True + + def wait(self, timeout=-1): + """mt.wait(timeout=-1) + + Wait for 0MQ to be done with the message or until `timeout`. + + Parameters + ---------- + timeout : float [default: -1, wait forever] + Maximum time in (s) to wait before raising NotDone. + + Returns + ------- + None + if done before `timeout` + + Raises + ------ + NotDone + if `timeout` reached before I am done. + """ + tic = time.time() + if timeout is False or timeout < 0: + remaining = 3600*24*7 # a week + else: + remaining = timeout + done = False + for evt in self.events: + if remaining < 0: + raise NotDone + evt.wait(timeout=remaining) + if not evt.is_set(): + raise NotDone + toc = time.time() + remaining -= (toc-tic) + tic = toc + + for peer in self.peers: + if remaining < 0: + raise NotDone + peer.wait(timeout=remaining) + toc = time.time() + remaining -= (toc-tic) + tic = toc + +__all__ = ['MessageTracker']
\ No newline at end of file diff --git a/scripts/external_libs/zmq/sugar/version.py b/scripts/external_libs/zmq/sugar/version.py new file mode 100644 index 00000000..ea8fbbc4 --- /dev/null +++ b/scripts/external_libs/zmq/sugar/version.py @@ -0,0 +1,48 @@ +"""PyZMQ and 0MQ version functions.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +from zmq.backend import zmq_version_info + + +VERSION_MAJOR = 14 +VERSION_MINOR = 5 +VERSION_PATCH = 0 +VERSION_EXTRA = "" +__version__ = '%i.%i.%i' % (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) + +if VERSION_EXTRA: + __version__ = "%s-%s" % (__version__, VERSION_EXTRA) + version_info = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH, float('inf')) +else: + version_info = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) + +__revision__ = '' + +def pyzmq_version(): + """return the version of pyzmq as a string""" + if __revision__: + return '@'.join([__version__,__revision__[:6]]) + else: + return __version__ + +def pyzmq_version_info(): + """return the pyzmq version as a tuple of at least three numbers + + If pyzmq is a development version, `inf` will be appended after the third integer. + """ + return version_info + + +def zmq_version(): + """return the version of libzmq as a string""" + return "%i.%i.%i" % zmq_version_info() + + +__all__ = ['zmq_version', 'zmq_version_info', + 'pyzmq_version','pyzmq_version_info', + '__version__', '__revision__' +] + diff --git a/scripts/external_libs/zmq/tests/__init__.py b/scripts/external_libs/zmq/tests/__init__.py new file mode 100644 index 00000000..325a3f19 --- /dev/null +++ b/scripts/external_libs/zmq/tests/__init__.py @@ -0,0 +1,211 @@ +# Copyright (c) PyZMQ Developers. +# Distributed under the terms of the Modified BSD License. + +import functools +import sys +import time +from threading import Thread + +from unittest import TestCase + +import zmq +from zmq.utils import jsonapi + +try: + import gevent + from zmq import green as gzmq + have_gevent = True +except ImportError: + have_gevent = False + +try: + from unittest import SkipTest +except ImportError: + try: + from nose import SkipTest + except ImportError: + class SkipTest(Exception): + pass + +PYPY = 'PyPy' in sys.version + +#----------------------------------------------------------------------------- +# skip decorators (directly from unittest) +#----------------------------------------------------------------------------- + +_id = lambda x: x + +def skip(reason): + """ + Unconditionally skip a test. + """ + def decorator(test_item): + if not (isinstance(test_item, type) and issubclass(test_item, TestCase)): + @functools.wraps(test_item) + def skip_wrapper(*args, **kwargs): + raise SkipTest(reason) + test_item = skip_wrapper + + test_item.__unittest_skip__ = True + test_item.__unittest_skip_why__ = reason + return test_item + return decorator + +def skip_if(condition, reason="Skipped"): + """ + Skip a test if the condition is true. + """ + if condition: + return skip(reason) + return _id + +skip_pypy = skip_if(PYPY, "Doesn't work on PyPy") + +#----------------------------------------------------------------------------- +# Base test class +#----------------------------------------------------------------------------- + +class BaseZMQTestCase(TestCase): + green = False + + @property + def Context(self): + if self.green: + return gzmq.Context + else: + return zmq.Context + + def socket(self, socket_type): + s = self.context.socket(socket_type) + self.sockets.append(s) + return s + + def setUp(self): + if self.green and not have_gevent: + raise SkipTest("requires gevent") + self.context = self.Context.instance() + self.sockets = [] + + def tearDown(self): + contexts = set([self.context]) + while self.sockets: + sock = self.sockets.pop() + contexts.add(sock.context) # in case additional contexts are created + sock.close(0) + for ctx in contexts: + t = Thread(target=ctx.term) + t.daemon = True + t.start() + t.join(timeout=2) + if t.is_alive(): + # reset Context.instance, so the failure to term doesn't corrupt subsequent tests + zmq.sugar.context.Context._instance = None + raise RuntimeError("context could not terminate, open sockets likely remain in test") + + def create_bound_pair(self, type1=zmq.PAIR, type2=zmq.PAIR, interface='tcp://127.0.0.1'): + """Create a bound socket pair using a random port.""" + s1 = self.context.socket(type1) + s1.setsockopt(zmq.LINGER, 0) + port = s1.bind_to_random_port(interface) + s2 = self.context.socket(type2) + s2.setsockopt(zmq.LINGER, 0) + s2.connect('%s:%s' % (interface, port)) + self.sockets.extend([s1,s2]) + return s1, s2 + + def ping_pong(self, s1, s2, msg): + s1.send(msg) + msg2 = s2.recv() + s2.send(msg2) + msg3 = s1.recv() + return msg3 + + def ping_pong_json(self, s1, s2, o): + if jsonapi.jsonmod is None: + raise SkipTest("No json library") + s1.send_json(o) + o2 = s2.recv_json() + s2.send_json(o2) + o3 = s1.recv_json() + return o3 + + def ping_pong_pyobj(self, s1, s2, o): + s1.send_pyobj(o) + o2 = s2.recv_pyobj() + s2.send_pyobj(o2) + o3 = s1.recv_pyobj() + return o3 + + def assertRaisesErrno(self, errno, func, *args, **kwargs): + try: + func(*args, **kwargs) + except zmq.ZMQError as e: + self.assertEqual(e.errno, errno, "wrong error raised, expected '%s' \ +got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) + else: + self.fail("Function did not raise any error") + + def _select_recv(self, multipart, socket, **kwargs): + """call recv[_multipart] in a way that raises if there is nothing to receive""" + if zmq.zmq_version_info() >= (3,1,0): + # zmq 3.1 has a bug, where poll can return false positives, + # so we wait a little bit just in case + # See LIBZMQ-280 on JIRA + time.sleep(0.1) + + r,w,x = zmq.select([socket], [], [], timeout=5) + assert len(r) > 0, "Should have received a message" + kwargs['flags'] = zmq.DONTWAIT | kwargs.get('flags', 0) + + recv = socket.recv_multipart if multipart else socket.recv + return recv(**kwargs) + + def recv(self, socket, **kwargs): + """call recv in a way that raises if there is nothing to receive""" + return self._select_recv(False, socket, **kwargs) + + def recv_multipart(self, socket, **kwargs): + """call recv_multipart in a way that raises if there is nothing to receive""" + return self._select_recv(True, socket, **kwargs) + + +class PollZMQTestCase(BaseZMQTestCase): + pass + +class GreenTest: + """Mixin for making green versions of test classes""" + green = True + + def assertRaisesErrno(self, errno, func, *args, **kwargs): + if errno == zmq.EAGAIN: + raise SkipTest("Skipping because we're green.") + try: + func(*args, **kwargs) + except zmq.ZMQError: + e = sys.exc_info()[1] + self.assertEqual(e.errno, errno, "wrong error raised, expected '%s' \ +got '%s'" % (zmq.ZMQError(errno), zmq.ZMQError(e.errno))) + else: + self.fail("Function did not raise any error") + + def tearDown(self): + contexts = set([self.context]) + while self.sockets: + sock = self.sockets.pop() + contexts.add(sock.context) # in case additional contexts are created + sock.close() + try: + gevent.joinall([gevent.spawn(ctx.term) for ctx in contexts], timeout=2, raise_error=True) + except gevent.Timeout: + raise RuntimeError("context could not terminate, open sockets likely remain in test") + + def skip_green(self): + raise SkipTest("Skipping because we are green") + +def skip_green(f): + def skipping_test(self, *args, **kwargs): + if self.green: + raise SkipTest("Skipping because we are green") + else: + return f(self, *args, **kwargs) + return skipping_test diff --git a/scripts/external_libs/zmq/tests/test_auth.py b/scripts/external_libs/zmq/tests/test_auth.py new file mode 100644 index 00000000..d350f61f --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_auth.py @@ -0,0 +1,431 @@ +# -*- coding: utf8 -*- + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import logging +import os +import shutil +import sys +import tempfile + +import zmq.auth +from zmq.auth.ioloop import IOLoopAuthenticator +from zmq.auth.thread import ThreadAuthenticator + +from zmq.eventloop import ioloop, zmqstream +from zmq.tests import (BaseZMQTestCase, SkipTest) + +class BaseAuthTestCase(BaseZMQTestCase): + def setUp(self): + if zmq.zmq_version_info() < (4,0): + raise SkipTest("security is new in libzmq 4.0") + try: + zmq.curve_keypair() + except zmq.ZMQError: + raise SkipTest("security requires libzmq to be linked against libsodium") + super(BaseAuthTestCase, self).setUp() + # enable debug logging while we run tests + logging.getLogger('zmq.auth').setLevel(logging.DEBUG) + self.auth = self.make_auth() + self.auth.start() + self.base_dir, self.public_keys_dir, self.secret_keys_dir = self.create_certs() + + def make_auth(self): + raise NotImplementedError() + + def tearDown(self): + if self.auth: + self.auth.stop() + self.auth = None + self.remove_certs(self.base_dir) + super(BaseAuthTestCase, self).tearDown() + + def create_certs(self): + """Create CURVE certificates for a test""" + + # Create temporary CURVE keypairs for this test run. We create all keys in a + # temp directory and then move them into the appropriate private or public + # directory. + + base_dir = tempfile.mkdtemp() + keys_dir = os.path.join(base_dir, 'certificates') + public_keys_dir = os.path.join(base_dir, 'public_keys') + secret_keys_dir = os.path.join(base_dir, 'private_keys') + + os.mkdir(keys_dir) + os.mkdir(public_keys_dir) + os.mkdir(secret_keys_dir) + + server_public_file, server_secret_file = zmq.auth.create_certificates(keys_dir, "server") + client_public_file, client_secret_file = zmq.auth.create_certificates(keys_dir, "client") + + for key_file in os.listdir(keys_dir): + if key_file.endswith(".key"): + shutil.move(os.path.join(keys_dir, key_file), + os.path.join(public_keys_dir, '.')) + + for key_file in os.listdir(keys_dir): + if key_file.endswith(".key_secret"): + shutil.move(os.path.join(keys_dir, key_file), + os.path.join(secret_keys_dir, '.')) + + return (base_dir, public_keys_dir, secret_keys_dir) + + def remove_certs(self, base_dir): + """Remove certificates for a test""" + shutil.rmtree(base_dir) + + def load_certs(self, secret_keys_dir): + """Return server and client certificate keys""" + server_secret_file = os.path.join(secret_keys_dir, "server.key_secret") + client_secret_file = os.path.join(secret_keys_dir, "client.key_secret") + + server_public, server_secret = zmq.auth.load_certificate(server_secret_file) + client_public, client_secret = zmq.auth.load_certificate(client_secret_file) + + return server_public, server_secret, client_public, client_secret + + +class TestThreadAuthentication(BaseAuthTestCase): + """Test authentication running in a thread""" + + def make_auth(self): + return ThreadAuthenticator(self.context) + + def can_connect(self, server, client): + """Check if client can connect to server using tcp transport""" + result = False + iface = 'tcp://127.0.0.1' + port = server.bind_to_random_port(iface) + client.connect("%s:%i" % (iface, port)) + msg = [b"Hello World"] + server.send_multipart(msg) + if client.poll(1000): + rcvd_msg = client.recv_multipart() + self.assertEqual(rcvd_msg, msg) + result = True + return result + + def test_null(self): + """threaded auth - NULL""" + # A default NULL connection should always succeed, and not + # go through our authentication infrastructure at all. + self.auth.stop() + self.auth = None + + server = self.socket(zmq.PUSH) + client = self.socket(zmq.PULL) + self.assertTrue(self.can_connect(server, client)) + + # By setting a domain we switch on authentication for NULL sockets, + # though no policies are configured yet. The client connection + # should still be allowed. + server = self.socket(zmq.PUSH) + server.zap_domain = b'global' + client = self.socket(zmq.PULL) + self.assertTrue(self.can_connect(server, client)) + + def test_blacklist(self): + """threaded auth - Blacklist""" + # Blacklist 127.0.0.1, connection should fail + self.auth.deny('127.0.0.1') + server = self.socket(zmq.PUSH) + # By setting a domain we switch on authentication for NULL sockets, + # though no policies are configured yet. + server.zap_domain = b'global' + client = self.socket(zmq.PULL) + self.assertFalse(self.can_connect(server, client)) + + def test_whitelist(self): + """threaded auth - Whitelist""" + # Whitelist 127.0.0.1, connection should pass" + self.auth.allow('127.0.0.1') + server = self.socket(zmq.PUSH) + # By setting a domain we switch on authentication for NULL sockets, + # though no policies are configured yet. + server.zap_domain = b'global' + client = self.socket(zmq.PULL) + self.assertTrue(self.can_connect(server, client)) + + def test_plain(self): + """threaded auth - PLAIN""" + + # Try PLAIN authentication - without configuring server, connection should fail + server = self.socket(zmq.PUSH) + server.plain_server = True + client = self.socket(zmq.PULL) + client.plain_username = b'admin' + client.plain_password = b'Password' + self.assertFalse(self.can_connect(server, client)) + + # Try PLAIN authentication - with server configured, connection should pass + server = self.socket(zmq.PUSH) + server.plain_server = True + client = self.socket(zmq.PULL) + client.plain_username = b'admin' + client.plain_password = b'Password' + self.auth.configure_plain(domain='*', passwords={'admin': 'Password'}) + self.assertTrue(self.can_connect(server, client)) + + # Try PLAIN authentication - with bogus credentials, connection should fail + server = self.socket(zmq.PUSH) + server.plain_server = True + client = self.socket(zmq.PULL) + client.plain_username = b'admin' + client.plain_password = b'Bogus' + self.assertFalse(self.can_connect(server, client)) + + # Remove authenticator and check that a normal connection works + self.auth.stop() + self.auth = None + + server = self.socket(zmq.PUSH) + client = self.socket(zmq.PULL) + self.assertTrue(self.can_connect(server, client)) + client.close() + server.close() + + def test_curve(self): + """threaded auth - CURVE""" + self.auth.allow('127.0.0.1') + certs = self.load_certs(self.secret_keys_dir) + server_public, server_secret, client_public, client_secret = certs + + #Try CURVE authentication - without configuring server, connection should fail + server = self.socket(zmq.PUSH) + server.curve_publickey = server_public + server.curve_secretkey = server_secret + server.curve_server = True + client = self.socket(zmq.PULL) + client.curve_publickey = client_public + client.curve_secretkey = client_secret + client.curve_serverkey = server_public + self.assertFalse(self.can_connect(server, client)) + + #Try CURVE authentication - with server configured to CURVE_ALLOW_ANY, connection should pass + self.auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY) + server = self.socket(zmq.PUSH) + server.curve_publickey = server_public + server.curve_secretkey = server_secret + server.curve_server = True + client = self.socket(zmq.PULL) + client.curve_publickey = client_public + client.curve_secretkey = client_secret + client.curve_serverkey = server_public + self.assertTrue(self.can_connect(server, client)) + + # Try CURVE authentication - with server configured, connection should pass + self.auth.configure_curve(domain='*', location=self.public_keys_dir) + server = self.socket(zmq.PUSH) + server.curve_publickey = server_public + server.curve_secretkey = server_secret + server.curve_server = True + client = self.socket(zmq.PULL) + client.curve_publickey = client_public + client.curve_secretkey = client_secret + client.curve_serverkey = server_public + self.assertTrue(self.can_connect(server, client)) + + # Remove authenticator and check that a normal connection works + self.auth.stop() + self.auth = None + + # Try connecting using NULL and no authentication enabled, connection should pass + server = self.socket(zmq.PUSH) + client = self.socket(zmq.PULL) + self.assertTrue(self.can_connect(server, client)) + + +def with_ioloop(method, expect_success=True): + """decorator for running tests with an IOLoop""" + def test_method(self): + r = method(self) + + loop = self.io_loop + if expect_success: + self.pullstream.on_recv(self.on_message_succeed) + else: + self.pullstream.on_recv(self.on_message_fail) + + t = loop.time() + loop.add_callback(self.attempt_connection) + loop.add_callback(self.send_msg) + if expect_success: + loop.add_timeout(t + 1, self.on_test_timeout_fail) + else: + loop.add_timeout(t + 1, self.on_test_timeout_succeed) + + loop.start() + if self.fail_msg: + self.fail(self.fail_msg) + + return r + return test_method + +def should_auth(method): + return with_ioloop(method, True) + +def should_not_auth(method): + return with_ioloop(method, False) + +class TestIOLoopAuthentication(BaseAuthTestCase): + """Test authentication running in ioloop""" + + def setUp(self): + self.fail_msg = None + self.io_loop = ioloop.IOLoop() + super(TestIOLoopAuthentication, self).setUp() + self.server = self.socket(zmq.PUSH) + self.client = self.socket(zmq.PULL) + self.pushstream = zmqstream.ZMQStream(self.server, self.io_loop) + self.pullstream = zmqstream.ZMQStream(self.client, self.io_loop) + + def make_auth(self): + return IOLoopAuthenticator(self.context, io_loop=self.io_loop) + + def tearDown(self): + if self.auth: + self.auth.stop() + self.auth = None + self.io_loop.close(all_fds=True) + super(TestIOLoopAuthentication, self).tearDown() + + def attempt_connection(self): + """Check if client can connect to server using tcp transport""" + iface = 'tcp://127.0.0.1' + port = self.server.bind_to_random_port(iface) + self.client.connect("%s:%i" % (iface, port)) + + def send_msg(self): + """Send a message from server to a client""" + msg = [b"Hello World"] + self.pushstream.send_multipart(msg) + + def on_message_succeed(self, frames): + """A message was received, as expected.""" + if frames != [b"Hello World"]: + self.fail_msg = "Unexpected message received" + self.io_loop.stop() + + def on_message_fail(self, frames): + """A message was received, unexpectedly.""" + self.fail_msg = 'Received messaged unexpectedly, security failed' + self.io_loop.stop() + + def on_test_timeout_succeed(self): + """Test timer expired, indicates test success""" + self.io_loop.stop() + + def on_test_timeout_fail(self): + """Test timer expired, indicates test failure""" + self.fail_msg = 'Test timed out' + self.io_loop.stop() + + @should_auth + def test_none(self): + """ioloop auth - NONE""" + # A default NULL connection should always succeed, and not + # go through our authentication infrastructure at all. + # no auth should be running + self.auth.stop() + self.auth = None + + @should_auth + def test_null(self): + """ioloop auth - NULL""" + # By setting a domain we switch on authentication for NULL sockets, + # though no policies are configured yet. The client connection + # should still be allowed. + self.server.zap_domain = b'global' + + @should_not_auth + def test_blacklist(self): + """ioloop auth - Blacklist""" + # Blacklist 127.0.0.1, connection should fail + self.auth.deny('127.0.0.1') + self.server.zap_domain = b'global' + + @should_auth + def test_whitelist(self): + """ioloop auth - Whitelist""" + # Whitelist 127.0.0.1, which overrides the blacklist, connection should pass" + self.auth.allow('127.0.0.1') + + self.server.setsockopt(zmq.ZAP_DOMAIN, b'global') + + @should_not_auth + def test_plain_unconfigured_server(self): + """ioloop auth - PLAIN, unconfigured server""" + self.client.plain_username = b'admin' + self.client.plain_password = b'Password' + # Try PLAIN authentication - without configuring server, connection should fail + self.server.plain_server = True + + @should_auth + def test_plain_configured_server(self): + """ioloop auth - PLAIN, configured server""" + self.client.plain_username = b'admin' + self.client.plain_password = b'Password' + # Try PLAIN authentication - with server configured, connection should pass + self.server.plain_server = True + self.auth.configure_plain(domain='*', passwords={'admin': 'Password'}) + + @should_not_auth + def test_plain_bogus_credentials(self): + """ioloop auth - PLAIN, bogus credentials""" + self.client.plain_username = b'admin' + self.client.plain_password = b'Bogus' + self.server.plain_server = True + + self.auth.configure_plain(domain='*', passwords={'admin': 'Password'}) + + @should_not_auth + def test_curve_unconfigured_server(self): + """ioloop auth - CURVE, unconfigured server""" + certs = self.load_certs(self.secret_keys_dir) + server_public, server_secret, client_public, client_secret = certs + + self.auth.allow('127.0.0.1') + + self.server.curve_publickey = server_public + self.server.curve_secretkey = server_secret + self.server.curve_server = True + + self.client.curve_publickey = client_public + self.client.curve_secretkey = client_secret + self.client.curve_serverkey = server_public + + @should_auth + def test_curve_allow_any(self): + """ioloop auth - CURVE, CURVE_ALLOW_ANY""" + certs = self.load_certs(self.secret_keys_dir) + server_public, server_secret, client_public, client_secret = certs + + self.auth.allow('127.0.0.1') + self.auth.configure_curve(domain='*', location=zmq.auth.CURVE_ALLOW_ANY) + + self.server.curve_publickey = server_public + self.server.curve_secretkey = server_secret + self.server.curve_server = True + + self.client.curve_publickey = client_public + self.client.curve_secretkey = client_secret + self.client.curve_serverkey = server_public + + @should_auth + def test_curve_configured_server(self): + """ioloop auth - CURVE, configured server""" + self.auth.allow('127.0.0.1') + certs = self.load_certs(self.secret_keys_dir) + server_public, server_secret, client_public, client_secret = certs + + self.auth.configure_curve(domain='*', location=self.public_keys_dir) + + self.server.curve_publickey = server_public + self.server.curve_secretkey = server_secret + self.server.curve_server = True + + self.client.curve_publickey = client_public + self.client.curve_secretkey = client_secret + self.client.curve_serverkey = server_public diff --git a/scripts/external_libs/zmq/tests/test_cffi_backend.py b/scripts/external_libs/zmq/tests/test_cffi_backend.py new file mode 100644 index 00000000..1f85eebf --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_cffi_backend.py @@ -0,0 +1,310 @@ +# -*- coding: utf8 -*- + +import sys +import time + +from unittest import TestCase + +from zmq.tests import BaseZMQTestCase, SkipTest + +try: + from zmq.backend.cffi import ( + zmq_version_info, + PUSH, PULL, IDENTITY, + REQ, REP, POLLIN, POLLOUT, + ) + from zmq.backend.cffi._cffi import ffi, C + have_ffi_backend = True +except ImportError: + have_ffi_backend = False + + +class TestCFFIBackend(TestCase): + + def setUp(self): + if not have_ffi_backend or not 'PyPy' in sys.version: + raise SkipTest('PyPy Tests Only') + + def test_zmq_version_info(self): + version = zmq_version_info() + + assert version[0] in range(2,11) + + def test_zmq_ctx_new_destroy(self): + ctx = C.zmq_ctx_new() + + assert ctx != ffi.NULL + assert 0 == C.zmq_ctx_destroy(ctx) + + def test_zmq_socket_open_close(self): + ctx = C.zmq_ctx_new() + socket = C.zmq_socket(ctx, PUSH) + + assert ctx != ffi.NULL + assert ffi.NULL != socket + assert 0 == C.zmq_close(socket) + assert 0 == C.zmq_ctx_destroy(ctx) + + def test_zmq_setsockopt(self): + ctx = C.zmq_ctx_new() + socket = C.zmq_socket(ctx, PUSH) + + identity = ffi.new('char[3]', 'zmq') + ret = C.zmq_setsockopt(socket, IDENTITY, ffi.cast('void*', identity), 3) + + assert ret == 0 + assert ctx != ffi.NULL + assert ffi.NULL != socket + assert 0 == C.zmq_close(socket) + assert 0 == C.zmq_ctx_destroy(ctx) + + def test_zmq_getsockopt(self): + ctx = C.zmq_ctx_new() + socket = C.zmq_socket(ctx, PUSH) + + identity = ffi.new('char[]', 'zmq') + ret = C.zmq_setsockopt(socket, IDENTITY, ffi.cast('void*', identity), 3) + assert ret == 0 + + option_len = ffi.new('size_t*', 3) + option = ffi.new('char*') + ret = C.zmq_getsockopt(socket, + IDENTITY, + ffi.cast('void*', option), + option_len) + + assert ret == 0 + assert ffi.string(ffi.cast('char*', option))[0] == "z" + assert ffi.string(ffi.cast('char*', option))[1] == "m" + assert ffi.string(ffi.cast('char*', option))[2] == "q" + assert ctx != ffi.NULL + assert ffi.NULL != socket + assert 0 == C.zmq_close(socket) + assert 0 == C.zmq_ctx_destroy(ctx) + + def test_zmq_bind(self): + ctx = C.zmq_ctx_new() + socket = C.zmq_socket(ctx, 8) + + assert 0 == C.zmq_bind(socket, 'tcp://*:4444') + assert ctx != ffi.NULL + assert ffi.NULL != socket + assert 0 == C.zmq_close(socket) + assert 0 == C.zmq_ctx_destroy(ctx) + + def test_zmq_bind_connect(self): + ctx = C.zmq_ctx_new() + + socket1 = C.zmq_socket(ctx, PUSH) + socket2 = C.zmq_socket(ctx, PULL) + + assert 0 == C.zmq_bind(socket1, 'tcp://*:4444') + assert 0 == C.zmq_connect(socket2, 'tcp://127.0.0.1:4444') + assert ctx != ffi.NULL + assert ffi.NULL != socket1 + assert ffi.NULL != socket2 + assert 0 == C.zmq_close(socket1) + assert 0 == C.zmq_close(socket2) + assert 0 == C.zmq_ctx_destroy(ctx) + + def test_zmq_msg_init_close(self): + zmq_msg = ffi.new('zmq_msg_t*') + + assert ffi.NULL != zmq_msg + assert 0 == C.zmq_msg_init(zmq_msg) + assert 0 == C.zmq_msg_close(zmq_msg) + + def test_zmq_msg_init_size(self): + zmq_msg = ffi.new('zmq_msg_t*') + + assert ffi.NULL != zmq_msg + assert 0 == C.zmq_msg_init_size(zmq_msg, 10) + assert 0 == C.zmq_msg_close(zmq_msg) + + def test_zmq_msg_init_data(self): + zmq_msg = ffi.new('zmq_msg_t*') + message = ffi.new('char[5]', 'Hello') + + assert 0 == C.zmq_msg_init_data(zmq_msg, + ffi.cast('void*', message), + 5, + ffi.NULL, + ffi.NULL) + + assert ffi.NULL != zmq_msg + assert 0 == C.zmq_msg_close(zmq_msg) + + def test_zmq_msg_data(self): + zmq_msg = ffi.new('zmq_msg_t*') + message = ffi.new('char[]', 'Hello') + assert 0 == C.zmq_msg_init_data(zmq_msg, + ffi.cast('void*', message), + 5, + ffi.NULL, + ffi.NULL) + + data = C.zmq_msg_data(zmq_msg) + + assert ffi.NULL != zmq_msg + assert ffi.string(ffi.cast("char*", data)) == 'Hello' + assert 0 == C.zmq_msg_close(zmq_msg) + + + def test_zmq_send(self): + ctx = C.zmq_ctx_new() + + sender = C.zmq_socket(ctx, REQ) + receiver = C.zmq_socket(ctx, REP) + + assert 0 == C.zmq_bind(receiver, 'tcp://*:7777') + assert 0 == C.zmq_connect(sender, 'tcp://127.0.0.1:7777') + + time.sleep(0.1) + + zmq_msg = ffi.new('zmq_msg_t*') + message = ffi.new('char[5]', 'Hello') + + C.zmq_msg_init_data(zmq_msg, + ffi.cast('void*', message), + ffi.cast('size_t', 5), + ffi.NULL, + ffi.NULL) + + assert 5 == C.zmq_msg_send(zmq_msg, sender, 0) + assert 0 == C.zmq_msg_close(zmq_msg) + assert C.zmq_close(sender) == 0 + assert C.zmq_close(receiver) == 0 + assert C.zmq_ctx_destroy(ctx) == 0 + + def test_zmq_recv(self): + ctx = C.zmq_ctx_new() + + sender = C.zmq_socket(ctx, REQ) + receiver = C.zmq_socket(ctx, REP) + + assert 0 == C.zmq_bind(receiver, 'tcp://*:2222') + assert 0 == C.zmq_connect(sender, 'tcp://127.0.0.1:2222') + + time.sleep(0.1) + + zmq_msg = ffi.new('zmq_msg_t*') + message = ffi.new('char[5]', 'Hello') + + C.zmq_msg_init_data(zmq_msg, + ffi.cast('void*', message), + ffi.cast('size_t', 5), + ffi.NULL, + ffi.NULL) + + zmq_msg2 = ffi.new('zmq_msg_t*') + C.zmq_msg_init(zmq_msg2) + + assert 5 == C.zmq_msg_send(zmq_msg, sender, 0) + assert 5 == C.zmq_msg_recv(zmq_msg2, receiver, 0) + assert 5 == C.zmq_msg_size(zmq_msg2) + assert b"Hello" == ffi.buffer(C.zmq_msg_data(zmq_msg2), + C.zmq_msg_size(zmq_msg2))[:] + assert C.zmq_close(sender) == 0 + assert C.zmq_close(receiver) == 0 + assert C.zmq_ctx_destroy(ctx) == 0 + + def test_zmq_poll(self): + ctx = C.zmq_ctx_new() + + sender = C.zmq_socket(ctx, REQ) + receiver = C.zmq_socket(ctx, REP) + + r1 = C.zmq_bind(receiver, 'tcp://*:3333') + r2 = C.zmq_connect(sender, 'tcp://127.0.0.1:3333') + + zmq_msg = ffi.new('zmq_msg_t*') + message = ffi.new('char[5]', 'Hello') + + C.zmq_msg_init_data(zmq_msg, + ffi.cast('void*', message), + ffi.cast('size_t', 5), + ffi.NULL, + ffi.NULL) + + receiver_pollitem = ffi.new('zmq_pollitem_t*') + receiver_pollitem.socket = receiver + receiver_pollitem.fd = 0 + receiver_pollitem.events = POLLIN | POLLOUT + receiver_pollitem.revents = 0 + + ret = C.zmq_poll(ffi.NULL, 0, 0) + assert ret == 0 + + ret = C.zmq_poll(receiver_pollitem, 1, 0) + assert ret == 0 + + ret = C.zmq_msg_send(zmq_msg, sender, 0) + print(ffi.string(C.zmq_strerror(C.zmq_errno()))) + assert ret == 5 + + time.sleep(0.2) + + ret = C.zmq_poll(receiver_pollitem, 1, 0) + assert ret == 1 + + assert int(receiver_pollitem.revents) & POLLIN + assert not int(receiver_pollitem.revents) & POLLOUT + + zmq_msg2 = ffi.new('zmq_msg_t*') + C.zmq_msg_init(zmq_msg2) + + ret_recv = C.zmq_msg_recv(zmq_msg2, receiver, 0) + assert ret_recv == 5 + + assert 5 == C.zmq_msg_size(zmq_msg2) + assert b"Hello" == ffi.buffer(C.zmq_msg_data(zmq_msg2), + C.zmq_msg_size(zmq_msg2))[:] + + sender_pollitem = ffi.new('zmq_pollitem_t*') + sender_pollitem.socket = sender + sender_pollitem.fd = 0 + sender_pollitem.events = POLLIN | POLLOUT + sender_pollitem.revents = 0 + + ret = C.zmq_poll(sender_pollitem, 1, 0) + assert ret == 0 + + zmq_msg_again = ffi.new('zmq_msg_t*') + message_again = ffi.new('char[11]', 'Hello Again') + + C.zmq_msg_init_data(zmq_msg_again, + ffi.cast('void*', message_again), + ffi.cast('size_t', 11), + ffi.NULL, + ffi.NULL) + + assert 11 == C.zmq_msg_send(zmq_msg_again, receiver, 0) + + time.sleep(0.2) + + assert 0 <= C.zmq_poll(sender_pollitem, 1, 0) + assert int(sender_pollitem.revents) & POLLIN + assert 11 == C.zmq_msg_recv(zmq_msg2, sender, 0) + assert 11 == C.zmq_msg_size(zmq_msg2) + assert b"Hello Again" == ffi.buffer(C.zmq_msg_data(zmq_msg2), + int(C.zmq_msg_size(zmq_msg2)))[:] + assert 0 == C.zmq_close(sender) + assert 0 == C.zmq_close(receiver) + assert 0 == C.zmq_ctx_destroy(ctx) + assert 0 == C.zmq_msg_close(zmq_msg) + assert 0 == C.zmq_msg_close(zmq_msg2) + assert 0 == C.zmq_msg_close(zmq_msg_again) + + def test_zmq_stopwatch_functions(self): + stopwatch = C.zmq_stopwatch_start() + ret = C.zmq_stopwatch_stop(stopwatch) + + assert ffi.NULL != stopwatch + assert 0 < int(ret) + + def test_zmq_sleep(self): + try: + C.zmq_sleep(1) + except Exception as e: + raise AssertionError("Error executing zmq_sleep(int)") + diff --git a/scripts/external_libs/zmq/tests/test_constants.py b/scripts/external_libs/zmq/tests/test_constants.py new file mode 100644 index 00000000..d32b2b48 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_constants.py @@ -0,0 +1,104 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import json +from unittest import TestCase + +import zmq + +from zmq.utils import constant_names +from zmq.sugar import constants as sugar_constants +from zmq.backend import constants as backend_constants + +all_set = set(constant_names.all_names) + +class TestConstants(TestCase): + + def _duplicate_test(self, namelist, listname): + """test that a given list has no duplicates""" + dupes = {} + for name in set(namelist): + cnt = namelist.count(name) + if cnt > 1: + dupes[name] = cnt + if dupes: + self.fail("The following names occur more than once in %s: %s" % (listname, json.dumps(dupes, indent=2))) + + def test_duplicate_all(self): + return self._duplicate_test(constant_names.all_names, "all_names") + + def _change_key(self, change, version): + """return changed-in key""" + return "%s-in %d.%d.%d" % tuple([change] + list(version)) + + def test_duplicate_changed(self): + all_changed = [] + for change in ("new", "removed"): + d = getattr(constant_names, change + "_in") + for version, namelist in d.items(): + all_changed.extend(namelist) + self._duplicate_test(namelist, self._change_key(change, version)) + + self._duplicate_test(all_changed, "all-changed") + + def test_changed_in_all(self): + missing = {} + for change in ("new", "removed"): + d = getattr(constant_names, change + "_in") + for version, namelist in d.items(): + key = self._change_key(change, version) + for name in namelist: + if name not in all_set: + if key not in missing: + missing[key] = [] + missing[key].append(name) + + if missing: + self.fail( + "The following names are missing in `all_names`: %s" % json.dumps(missing, indent=2) + ) + + def test_no_negative_constants(self): + for name in sugar_constants.__all__: + self.assertNotEqual(getattr(zmq, name), sugar_constants._UNDEFINED) + + def test_undefined_constants(self): + all_aliases = [] + for alias_group in sugar_constants.aliases: + all_aliases.extend(alias_group) + + for name in all_set.difference(all_aliases): + raw = getattr(backend_constants, name) + if raw == sugar_constants._UNDEFINED: + self.assertRaises(AttributeError, getattr, zmq, name) + else: + self.assertEqual(getattr(zmq, name), raw) + + def test_new(self): + zmq_version = zmq.zmq_version_info() + for version, new_names in constant_names.new_in.items(): + should_have = zmq_version >= version + for name in new_names: + try: + value = getattr(zmq, name) + except AttributeError: + if should_have: + self.fail("AttributeError: zmq.%s" % name) + else: + if not should_have: + self.fail("Shouldn't have: zmq.%s=%s" % (name, value)) + + def test_removed(self): + zmq_version = zmq.zmq_version_info() + for version, new_names in constant_names.removed_in.items(): + should_have = zmq_version < version + for name in new_names: + try: + value = getattr(zmq, name) + except AttributeError: + if should_have: + self.fail("AttributeError: zmq.%s" % name) + else: + if not should_have: + self.fail("Shouldn't have: zmq.%s=%s" % (name, value)) + diff --git a/scripts/external_libs/zmq/tests/test_context.py b/scripts/external_libs/zmq/tests/test_context.py new file mode 100644 index 00000000..e3280778 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_context.py @@ -0,0 +1,257 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import gc +import sys +import time +from threading import Thread, Event + +import zmq +from zmq.tests import ( + BaseZMQTestCase, have_gevent, GreenTest, skip_green, PYPY, SkipTest, +) + + +class TestContext(BaseZMQTestCase): + + def test_init(self): + c1 = self.Context() + self.assert_(isinstance(c1, self.Context)) + del c1 + c2 = self.Context() + self.assert_(isinstance(c2, self.Context)) + del c2 + c3 = self.Context() + self.assert_(isinstance(c3, self.Context)) + del c3 + + def test_dir(self): + ctx = self.Context() + self.assertTrue('socket' in dir(ctx)) + if zmq.zmq_version_info() > (3,): + self.assertTrue('IO_THREADS' in dir(ctx)) + ctx.term() + + def test_term(self): + c = self.Context() + c.term() + self.assert_(c.closed) + + def test_context_manager(self): + with self.Context() as c: + pass + self.assert_(c.closed) + + def test_fail_init(self): + self.assertRaisesErrno(zmq.EINVAL, self.Context, -1) + + def test_term_hang(self): + rep,req = self.create_bound_pair(zmq.ROUTER, zmq.DEALER) + req.setsockopt(zmq.LINGER, 0) + req.send(b'hello', copy=False) + req.close() + rep.close() + self.context.term() + + def test_instance(self): + ctx = self.Context.instance() + c2 = self.Context.instance(io_threads=2) + self.assertTrue(c2 is ctx) + c2.term() + c3 = self.Context.instance() + c4 = self.Context.instance() + self.assertFalse(c3 is c2) + self.assertFalse(c3.closed) + self.assertTrue(c3 is c4) + + def test_many_sockets(self): + """opening and closing many sockets shouldn't cause problems""" + ctx = self.Context() + for i in range(16): + sockets = [ ctx.socket(zmq.REP) for i in range(65) ] + [ s.close() for s in sockets ] + # give the reaper a chance + time.sleep(1e-2) + ctx.term() + + def test_sockopts(self): + """setting socket options with ctx attributes""" + ctx = self.Context() + ctx.linger = 5 + self.assertEqual(ctx.linger, 5) + s = ctx.socket(zmq.REQ) + self.assertEqual(s.linger, 5) + self.assertEqual(s.getsockopt(zmq.LINGER), 5) + s.close() + # check that subscribe doesn't get set on sockets that don't subscribe: + ctx.subscribe = b'' + s = ctx.socket(zmq.REQ) + s.close() + + ctx.term() + + + def test_destroy(self): + """Context.destroy should close sockets""" + ctx = self.Context() + sockets = [ ctx.socket(zmq.REP) for i in range(65) ] + + # close half of the sockets + [ s.close() for s in sockets[::2] ] + + ctx.destroy() + # reaper is not instantaneous + time.sleep(1e-2) + for s in sockets: + self.assertTrue(s.closed) + + def test_destroy_linger(self): + """Context.destroy should set linger on closing sockets""" + req,rep = self.create_bound_pair(zmq.REQ, zmq.REP) + req.send(b'hi') + time.sleep(1e-2) + self.context.destroy(linger=0) + # reaper is not instantaneous + time.sleep(1e-2) + for s in (req,rep): + self.assertTrue(s.closed) + + def test_term_noclose(self): + """Context.term won't close sockets""" + ctx = self.Context() + s = ctx.socket(zmq.REQ) + self.assertFalse(s.closed) + t = Thread(target=ctx.term) + t.start() + t.join(timeout=0.1) + self.assertTrue(t.is_alive(), "Context should be waiting") + s.close() + t.join(timeout=0.1) + self.assertFalse(t.is_alive(), "Context should have closed") + + def test_gc(self): + """test close&term by garbage collection alone""" + if PYPY: + raise SkipTest("GC doesn't work ") + + # test credit @dln (GH #137): + def gcf(): + def inner(): + ctx = self.Context() + s = ctx.socket(zmq.PUSH) + inner() + gc.collect() + t = Thread(target=gcf) + t.start() + t.join(timeout=1) + self.assertFalse(t.is_alive(), "Garbage collection should have cleaned up context") + + def test_cyclic_destroy(self): + """ctx.destroy should succeed when cyclic ref prevents gc""" + # test credit @dln (GH #137): + class CyclicReference(object): + def __init__(self, parent=None): + self.parent = parent + + def crash(self, sock): + self.sock = sock + self.child = CyclicReference(self) + + def crash_zmq(): + ctx = self.Context() + sock = ctx.socket(zmq.PULL) + c = CyclicReference() + c.crash(sock) + ctx.destroy() + + crash_zmq() + + def test_term_thread(self): + """ctx.term should not crash active threads (#139)""" + ctx = self.Context() + evt = Event() + evt.clear() + + def block(): + s = ctx.socket(zmq.REP) + s.bind_to_random_port('tcp://127.0.0.1') + evt.set() + try: + s.recv() + except zmq.ZMQError as e: + self.assertEqual(e.errno, zmq.ETERM) + return + finally: + s.close() + self.fail("recv should have been interrupted with ETERM") + t = Thread(target=block) + t.start() + + evt.wait(1) + self.assertTrue(evt.is_set(), "sync event never fired") + time.sleep(0.01) + ctx.term() + t.join(timeout=1) + self.assertFalse(t.is_alive(), "term should have interrupted s.recv()") + + def test_destroy_no_sockets(self): + ctx = self.Context() + s = ctx.socket(zmq.PUB) + s.bind_to_random_port('tcp://127.0.0.1') + s.close() + ctx.destroy() + assert s.closed + assert ctx.closed + + def test_ctx_opts(self): + if zmq.zmq_version_info() < (3,): + raise SkipTest("context options require libzmq 3") + ctx = self.Context() + ctx.set(zmq.MAX_SOCKETS, 2) + self.assertEqual(ctx.get(zmq.MAX_SOCKETS), 2) + ctx.max_sockets = 100 + self.assertEqual(ctx.max_sockets, 100) + self.assertEqual(ctx.get(zmq.MAX_SOCKETS), 100) + + def test_shadow(self): + ctx = self.Context() + ctx2 = self.Context.shadow(ctx.underlying) + self.assertEqual(ctx.underlying, ctx2.underlying) + s = ctx.socket(zmq.PUB) + s.close() + del ctx2 + self.assertFalse(ctx.closed) + s = ctx.socket(zmq.PUB) + ctx2 = self.Context.shadow(ctx.underlying) + s2 = ctx2.socket(zmq.PUB) + s.close() + s2.close() + ctx.term() + self.assertRaisesErrno(zmq.EFAULT, ctx2.socket, zmq.PUB) + del ctx2 + + def test_shadow_pyczmq(self): + try: + from pyczmq import zctx, zsocket, zstr + except Exception: + raise SkipTest("Requires pyczmq") + + ctx = zctx.new() + a = zsocket.new(ctx, zmq.PUSH) + zsocket.bind(a, "inproc://a") + ctx2 = self.Context.shadow_pyczmq(ctx) + b = ctx2.socket(zmq.PULL) + b.connect("inproc://a") + zstr.send(a, b'hi') + rcvd = self.recv(b) + self.assertEqual(rcvd, b'hi') + b.close() + + +if False: # disable green context tests + class TestContextGreen(GreenTest, TestContext): + """gevent subclass of context tests""" + # skip tests that use real threads: + test_gc = GreenTest.skip_green + test_term_thread = GreenTest.skip_green + test_destroy_linger = GreenTest.skip_green diff --git a/scripts/external_libs/zmq/tests/test_device.py b/scripts/external_libs/zmq/tests/test_device.py new file mode 100644 index 00000000..f8305074 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_device.py @@ -0,0 +1,146 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import time + +import zmq +from zmq import devices +from zmq.tests import BaseZMQTestCase, SkipTest, have_gevent, GreenTest, PYPY +from zmq.utils.strtypes import (bytes,unicode,basestring) + +if PYPY: + # cleanup of shared Context doesn't work on PyPy + devices.Device.context_factory = zmq.Context + +class TestDevice(BaseZMQTestCase): + + def test_device_types(self): + for devtype in (zmq.STREAMER, zmq.FORWARDER, zmq.QUEUE): + dev = devices.Device(devtype, zmq.PAIR, zmq.PAIR) + self.assertEqual(dev.device_type, devtype) + del dev + + def test_device_attributes(self): + dev = devices.Device(zmq.QUEUE, zmq.SUB, zmq.PUB) + self.assertEqual(dev.in_type, zmq.SUB) + self.assertEqual(dev.out_type, zmq.PUB) + self.assertEqual(dev.device_type, zmq.QUEUE) + self.assertEqual(dev.daemon, True) + del dev + + def test_tsdevice_attributes(self): + dev = devices.Device(zmq.QUEUE, zmq.SUB, zmq.PUB) + self.assertEqual(dev.in_type, zmq.SUB) + self.assertEqual(dev.out_type, zmq.PUB) + self.assertEqual(dev.device_type, zmq.QUEUE) + self.assertEqual(dev.daemon, True) + del dev + + + def test_single_socket_forwarder_connect(self): + dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1) + req = self.context.socket(zmq.REQ) + port = req.bind_to_random_port('tcp://127.0.0.1') + dev.connect_in('tcp://127.0.0.1:%i'%port) + dev.start() + time.sleep(.25) + msg = b'hello' + req.send(msg) + self.assertEqual(msg, self.recv(req)) + del dev + req.close() + dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1) + req = self.context.socket(zmq.REQ) + port = req.bind_to_random_port('tcp://127.0.0.1') + dev.connect_out('tcp://127.0.0.1:%i'%port) + dev.start() + time.sleep(.25) + msg = b'hello again' + req.send(msg) + self.assertEqual(msg, self.recv(req)) + del dev + req.close() + + def test_single_socket_forwarder_bind(self): + dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1) + # select random port: + binder = self.context.socket(zmq.REQ) + port = binder.bind_to_random_port('tcp://127.0.0.1') + binder.close() + time.sleep(0.1) + req = self.context.socket(zmq.REQ) + req.connect('tcp://127.0.0.1:%i'%port) + dev.bind_in('tcp://127.0.0.1:%i'%port) + dev.start() + time.sleep(.25) + msg = b'hello' + req.send(msg) + self.assertEqual(msg, self.recv(req)) + del dev + req.close() + dev = devices.ThreadDevice(zmq.QUEUE, zmq.REP, -1) + # select random port: + binder = self.context.socket(zmq.REQ) + port = binder.bind_to_random_port('tcp://127.0.0.1') + binder.close() + time.sleep(0.1) + req = self.context.socket(zmq.REQ) + req.connect('tcp://127.0.0.1:%i'%port) + dev.bind_in('tcp://127.0.0.1:%i'%port) + dev.start() + time.sleep(.25) + msg = b'hello again' + req.send(msg) + self.assertEqual(msg, self.recv(req)) + del dev + req.close() + + def test_proxy(self): + if zmq.zmq_version_info() < (3,2): + raise SkipTest("Proxies only in libzmq >= 3") + dev = devices.ThreadProxy(zmq.PULL, zmq.PUSH, zmq.PUSH) + binder = self.context.socket(zmq.REQ) + iface = 'tcp://127.0.0.1' + port = binder.bind_to_random_port(iface) + port2 = binder.bind_to_random_port(iface) + port3 = binder.bind_to_random_port(iface) + binder.close() + time.sleep(0.1) + dev.bind_in("%s:%i" % (iface, port)) + dev.bind_out("%s:%i" % (iface, port2)) + dev.bind_mon("%s:%i" % (iface, port3)) + dev.start() + time.sleep(0.25) + msg = b'hello' + push = self.context.socket(zmq.PUSH) + push.connect("%s:%i" % (iface, port)) + pull = self.context.socket(zmq.PULL) + pull.connect("%s:%i" % (iface, port2)) + mon = self.context.socket(zmq.PULL) + mon.connect("%s:%i" % (iface, port3)) + push.send(msg) + self.sockets.extend([push, pull, mon]) + self.assertEqual(msg, self.recv(pull)) + self.assertEqual(msg, self.recv(mon)) + +if have_gevent: + import gevent + import zmq.green + + class TestDeviceGreen(GreenTest, BaseZMQTestCase): + + def test_green_device(self): + rep = self.context.socket(zmq.REP) + req = self.context.socket(zmq.REQ) + self.sockets.extend([req, rep]) + port = rep.bind_to_random_port('tcp://127.0.0.1') + g = gevent.spawn(zmq.green.device, zmq.QUEUE, rep, rep) + req.connect('tcp://127.0.0.1:%i' % port) + req.send(b'hi') + timeout = gevent.Timeout(3) + timeout.start() + receiver = gevent.spawn(req.recv) + self.assertEqual(receiver.get(2), b'hi') + timeout.cancel() + g.kill(block=True) + diff --git a/scripts/external_libs/zmq/tests/test_error.py b/scripts/external_libs/zmq/tests/test_error.py new file mode 100644 index 00000000..a2eee14a --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_error.py @@ -0,0 +1,43 @@ +# -*- coding: utf8 -*- +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import sys +import time + +import zmq +from zmq import ZMQError, strerror, Again, ContextTerminated +from zmq.tests import BaseZMQTestCase + +if sys.version_info[0] >= 3: + long = int + +class TestZMQError(BaseZMQTestCase): + + def test_strerror(self): + """test that strerror gets the right type.""" + for i in range(10): + e = strerror(i) + self.assertTrue(isinstance(e, str)) + + def test_zmqerror(self): + for errno in range(10): + e = ZMQError(errno) + self.assertEqual(e.errno, errno) + self.assertEqual(str(e), strerror(errno)) + + def test_again(self): + s = self.context.socket(zmq.REP) + self.assertRaises(Again, s.recv, zmq.NOBLOCK) + self.assertRaisesErrno(zmq.EAGAIN, s.recv, zmq.NOBLOCK) + s.close() + + def atest_ctxterm(self): + s = self.context.socket(zmq.REP) + t = Thread(target=self.context.term) + t.start() + self.assertRaises(ContextTerminated, s.recv, zmq.NOBLOCK) + self.assertRaisesErrno(zmq.TERM, s.recv, zmq.NOBLOCK) + s.close() + t.join() + diff --git a/scripts/external_libs/zmq/tests/test_etc.py b/scripts/external_libs/zmq/tests/test_etc.py new file mode 100644 index 00000000..ad224064 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_etc.py @@ -0,0 +1,15 @@ +# Copyright (c) PyZMQ Developers. +# Distributed under the terms of the Modified BSD License. + +import sys + +import zmq + +from . import skip_if + +@skip_if(zmq.zmq_version_info() < (4,1), "libzmq < 4.1") +def test_has(): + assert not zmq.has('something weird') + has_ipc = zmq.has('ipc') + not_windows = not sys.platform.startswith('win') + assert has_ipc == not_windows diff --git a/scripts/external_libs/zmq/tests/test_imports.py b/scripts/external_libs/zmq/tests/test_imports.py new file mode 100644 index 00000000..c0ddfaac --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_imports.py @@ -0,0 +1,62 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import sys +from unittest import TestCase + +class TestImports(TestCase): + """Test Imports - the quickest test to ensure that we haven't + introduced version-incompatible syntax errors.""" + + def test_toplevel(self): + """test toplevel import""" + import zmq + + def test_core(self): + """test core imports""" + from zmq import Context + from zmq import Socket + from zmq import Poller + from zmq import Frame + from zmq import constants + from zmq import device, proxy + from zmq import Stopwatch + from zmq import ( + zmq_version, + zmq_version_info, + pyzmq_version, + pyzmq_version_info, + ) + + def test_devices(self): + """test device imports""" + import zmq.devices + from zmq.devices import basedevice + from zmq.devices import monitoredqueue + from zmq.devices import monitoredqueuedevice + + def test_log(self): + """test log imports""" + import zmq.log + from zmq.log import handlers + + def test_eventloop(self): + """test eventloop imports""" + import zmq.eventloop + from zmq.eventloop import ioloop + from zmq.eventloop import zmqstream + from zmq.eventloop.minitornado.platform import auto + from zmq.eventloop.minitornado import ioloop + + def test_utils(self): + """test util imports""" + import zmq.utils + from zmq.utils import strtypes + from zmq.utils import jsonapi + + def test_ssh(self): + """test ssh imports""" + from zmq.ssh import tunnel + + + diff --git a/scripts/external_libs/zmq/tests/test_ioloop.py b/scripts/external_libs/zmq/tests/test_ioloop.py new file mode 100644 index 00000000..2a8b1153 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_ioloop.py @@ -0,0 +1,113 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import time +import os +import threading + +import zmq +from zmq.tests import BaseZMQTestCase +from zmq.eventloop import ioloop +from zmq.eventloop.minitornado.ioloop import _Timeout +try: + from tornado.ioloop import PollIOLoop, IOLoop as BaseIOLoop +except ImportError: + from zmq.eventloop.minitornado.ioloop import IOLoop as BaseIOLoop + + +def printer(): + os.system("say hello") + raise Exception + print (time.time()) + + +class Delay(threading.Thread): + def __init__(self, f, delay=1): + self.f=f + self.delay=delay + self.aborted=False + self.cond=threading.Condition() + super(Delay, self).__init__() + + def run(self): + self.cond.acquire() + self.cond.wait(self.delay) + self.cond.release() + if not self.aborted: + self.f() + + def abort(self): + self.aborted=True + self.cond.acquire() + self.cond.notify() + self.cond.release() + + +class TestIOLoop(BaseZMQTestCase): + + def test_simple(self): + """simple IOLoop creation test""" + loop = ioloop.IOLoop() + dc = ioloop.PeriodicCallback(loop.stop, 200, loop) + pc = ioloop.PeriodicCallback(lambda : None, 10, loop) + pc.start() + dc.start() + t = Delay(loop.stop,1) + t.start() + loop.start() + if t.isAlive(): + t.abort() + else: + self.fail("IOLoop failed to exit") + + def test_timeout_compare(self): + """test timeout comparisons""" + loop = ioloop.IOLoop() + t = _Timeout(1, 2, loop) + t2 = _Timeout(1, 3, loop) + self.assertEqual(t < t2, id(t) < id(t2)) + t2 = _Timeout(2,1, loop) + self.assertTrue(t < t2) + + def test_poller_events(self): + """Tornado poller implementation maps events correctly""" + req,rep = self.create_bound_pair(zmq.REQ, zmq.REP) + poller = ioloop.ZMQPoller() + poller.register(req, ioloop.IOLoop.READ) + poller.register(rep, ioloop.IOLoop.READ) + events = dict(poller.poll(0)) + self.assertEqual(events.get(rep), None) + self.assertEqual(events.get(req), None) + + poller.register(req, ioloop.IOLoop.WRITE) + poller.register(rep, ioloop.IOLoop.WRITE) + events = dict(poller.poll(1)) + self.assertEqual(events.get(req), ioloop.IOLoop.WRITE) + self.assertEqual(events.get(rep), None) + + poller.register(rep, ioloop.IOLoop.READ) + req.send(b'hi') + events = dict(poller.poll(1)) + self.assertEqual(events.get(rep), ioloop.IOLoop.READ) + self.assertEqual(events.get(req), None) + + def test_instance(self): + """Test IOLoop.instance returns the right object""" + loop = ioloop.IOLoop.instance() + self.assertEqual(loop.__class__, ioloop.IOLoop) + loop = BaseIOLoop.instance() + self.assertEqual(loop.__class__, ioloop.IOLoop) + + def test_close_all(self): + """Test close(all_fds=True)""" + loop = ioloop.IOLoop.instance() + req,rep = self.create_bound_pair(zmq.REQ, zmq.REP) + loop.add_handler(req, lambda msg: msg, ioloop.IOLoop.READ) + loop.add_handler(rep, lambda msg: msg, ioloop.IOLoop.READ) + self.assertEqual(req.closed, False) + self.assertEqual(rep.closed, False) + loop.close(all_fds=True) + self.assertEqual(req.closed, True) + self.assertEqual(rep.closed, True) + + diff --git a/scripts/external_libs/zmq/tests/test_log.py b/scripts/external_libs/zmq/tests/test_log.py new file mode 100644 index 00000000..9206f095 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_log.py @@ -0,0 +1,116 @@ +# encoding: utf-8 + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import logging +import time +from unittest import TestCase + +import zmq +from zmq.log import handlers +from zmq.utils.strtypes import b, u +from zmq.tests import BaseZMQTestCase + + +class TestPubLog(BaseZMQTestCase): + + iface = 'inproc://zmqlog' + topic= 'zmq' + + @property + def logger(self): + # print dir(self) + logger = logging.getLogger('zmqtest') + logger.setLevel(logging.DEBUG) + return logger + + def connect_handler(self, topic=None): + topic = self.topic if topic is None else topic + logger = self.logger + pub,sub = self.create_bound_pair(zmq.PUB, zmq.SUB) + handler = handlers.PUBHandler(pub) + handler.setLevel(logging.DEBUG) + handler.root_topic = topic + logger.addHandler(handler) + sub.setsockopt(zmq.SUBSCRIBE, b(topic)) + time.sleep(0.1) + return logger, handler, sub + + def test_init_iface(self): + logger = self.logger + ctx = self.context + handler = handlers.PUBHandler(self.iface) + self.assertFalse(handler.ctx is ctx) + self.sockets.append(handler.socket) + # handler.ctx.term() + handler = handlers.PUBHandler(self.iface, self.context) + self.sockets.append(handler.socket) + self.assertTrue(handler.ctx is ctx) + handler.setLevel(logging.DEBUG) + handler.root_topic = self.topic + logger.addHandler(handler) + sub = ctx.socket(zmq.SUB) + self.sockets.append(sub) + sub.setsockopt(zmq.SUBSCRIBE, b(self.topic)) + sub.connect(self.iface) + import time; time.sleep(0.25) + msg1 = 'message' + logger.info(msg1) + + (topic, msg2) = sub.recv_multipart() + self.assertEqual(topic, b'zmq.INFO') + self.assertEqual(msg2, b(msg1)+b'\n') + logger.removeHandler(handler) + + def test_init_socket(self): + pub,sub = self.create_bound_pair(zmq.PUB, zmq.SUB) + logger = self.logger + handler = handlers.PUBHandler(pub) + handler.setLevel(logging.DEBUG) + handler.root_topic = self.topic + logger.addHandler(handler) + + self.assertTrue(handler.socket is pub) + self.assertTrue(handler.ctx is pub.context) + self.assertTrue(handler.ctx is self.context) + sub.setsockopt(zmq.SUBSCRIBE, b(self.topic)) + import time; time.sleep(0.1) + msg1 = 'message' + logger.info(msg1) + + (topic, msg2) = sub.recv_multipart() + self.assertEqual(topic, b'zmq.INFO') + self.assertEqual(msg2, b(msg1)+b'\n') + logger.removeHandler(handler) + + def test_root_topic(self): + logger, handler, sub = self.connect_handler() + handler.socket.bind(self.iface) + sub2 = sub.context.socket(zmq.SUB) + self.sockets.append(sub2) + sub2.connect(self.iface) + sub2.setsockopt(zmq.SUBSCRIBE, b'') + handler.root_topic = b'twoonly' + msg1 = 'ignored' + logger.info(msg1) + self.assertRaisesErrno(zmq.EAGAIN, sub.recv, zmq.NOBLOCK) + topic,msg2 = sub2.recv_multipart() + self.assertEqual(topic, b'twoonly.INFO') + self.assertEqual(msg2, b(msg1)+b'\n') + + logger.removeHandler(handler) + + def test_unicode_message(self): + logger, handler, sub = self.connect_handler() + base_topic = b(self.topic + '.INFO') + for msg, expected in [ + (u('hello'), [base_topic, b('hello\n')]), + (u('héllo'), [base_topic, b('héllo\n')]), + (u('tøpic::héllo'), [base_topic + b('.tøpic'), b('héllo\n')]), + ]: + logger.info(msg) + received = sub.recv_multipart() + self.assertEqual(received, expected) + diff --git a/scripts/external_libs/zmq/tests/test_message.py b/scripts/external_libs/zmq/tests/test_message.py new file mode 100644 index 00000000..d8770bdf --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_message.py @@ -0,0 +1,362 @@ +# -*- coding: utf8 -*- +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import copy +import sys +try: + from sys import getrefcount as grc +except ImportError: + grc = None + +import time +from pprint import pprint +from unittest import TestCase + +import zmq +from zmq.tests import BaseZMQTestCase, SkipTest, skip_pypy, PYPY +from zmq.utils.strtypes import unicode, bytes, b, u + + +# some useful constants: + +x = b'x' + +try: + view = memoryview +except NameError: + view = buffer + +if grc: + rc0 = grc(x) + v = view(x) + view_rc = grc(x) - rc0 + +def await_gc(obj, rc): + """wait for refcount on an object to drop to an expected value + + Necessary because of the zero-copy gc thread, + which can take some time to receive its DECREF message. + """ + for i in range(50): + # rc + 2 because of the refs in this function + if grc(obj) <= rc + 2: + return + time.sleep(0.05) + +class TestFrame(BaseZMQTestCase): + + @skip_pypy + def test_above_30(self): + """Message above 30 bytes are never copied by 0MQ.""" + for i in range(5, 16): # 32, 64,..., 65536 + s = (2**i)*x + self.assertEqual(grc(s), 2) + m = zmq.Frame(s) + self.assertEqual(grc(s), 4) + del m + await_gc(s, 2) + self.assertEqual(grc(s), 2) + del s + + def test_str(self): + """Test the str representations of the Frames.""" + for i in range(16): + s = (2**i)*x + m = zmq.Frame(s) + m_str = str(m) + m_str_b = b(m_str) # py3compat + self.assertEqual(s, m_str_b) + + def test_bytes(self): + """Test the Frame.bytes property.""" + for i in range(1,16): + s = (2**i)*x + m = zmq.Frame(s) + b = m.bytes + self.assertEqual(s, m.bytes) + if not PYPY: + # check that it copies + self.assert_(b is not s) + # check that it copies only once + self.assert_(b is m.bytes) + + def test_unicode(self): + """Test the unicode representations of the Frames.""" + s = u('asdf') + self.assertRaises(TypeError, zmq.Frame, s) + for i in range(16): + s = (2**i)*u('§') + m = zmq.Frame(s.encode('utf8')) + self.assertEqual(s, unicode(m.bytes,'utf8')) + + def test_len(self): + """Test the len of the Frames.""" + for i in range(16): + s = (2**i)*x + m = zmq.Frame(s) + self.assertEqual(len(s), len(m)) + + @skip_pypy + def test_lifecycle1(self): + """Run through a ref counting cycle with a copy.""" + for i in range(5, 16): # 32, 64,..., 65536 + s = (2**i)*x + rc = 2 + self.assertEqual(grc(s), rc) + m = zmq.Frame(s) + rc += 2 + self.assertEqual(grc(s), rc) + m2 = copy.copy(m) + rc += 1 + self.assertEqual(grc(s), rc) + buf = m2.buffer + + rc += view_rc + self.assertEqual(grc(s), rc) + + self.assertEqual(s, b(str(m))) + self.assertEqual(s, bytes(m2)) + self.assertEqual(s, m.bytes) + # self.assert_(s is str(m)) + # self.assert_(s is str(m2)) + del m2 + rc -= 1 + self.assertEqual(grc(s), rc) + rc -= view_rc + del buf + self.assertEqual(grc(s), rc) + del m + rc -= 2 + await_gc(s, rc) + self.assertEqual(grc(s), rc) + self.assertEqual(rc, 2) + del s + + @skip_pypy + def test_lifecycle2(self): + """Run through a different ref counting cycle with a copy.""" + for i in range(5, 16): # 32, 64,..., 65536 + s = (2**i)*x + rc = 2 + self.assertEqual(grc(s), rc) + m = zmq.Frame(s) + rc += 2 + self.assertEqual(grc(s), rc) + m2 = copy.copy(m) + rc += 1 + self.assertEqual(grc(s), rc) + buf = m.buffer + rc += view_rc + self.assertEqual(grc(s), rc) + self.assertEqual(s, b(str(m))) + self.assertEqual(s, bytes(m2)) + self.assertEqual(s, m2.bytes) + self.assertEqual(s, m.bytes) + # self.assert_(s is str(m)) + # self.assert_(s is str(m2)) + del buf + self.assertEqual(grc(s), rc) + del m + # m.buffer is kept until m is del'd + rc -= view_rc + rc -= 1 + self.assertEqual(grc(s), rc) + del m2 + rc -= 2 + await_gc(s, rc) + self.assertEqual(grc(s), rc) + self.assertEqual(rc, 2) + del s + + @skip_pypy + def test_tracker(self): + m = zmq.Frame(b'asdf', track=True) + self.assertFalse(m.tracker.done) + pm = zmq.MessageTracker(m) + self.assertFalse(pm.done) + del m + for i in range(10): + if pm.done: + break + time.sleep(0.1) + self.assertTrue(pm.done) + + def test_no_tracker(self): + m = zmq.Frame(b'asdf', track=False) + self.assertEqual(m.tracker, None) + m2 = copy.copy(m) + self.assertEqual(m2.tracker, None) + self.assertRaises(ValueError, zmq.MessageTracker, m) + + @skip_pypy + def test_multi_tracker(self): + m = zmq.Frame(b'asdf', track=True) + m2 = zmq.Frame(b'whoda', track=True) + mt = zmq.MessageTracker(m,m2) + self.assertFalse(m.tracker.done) + self.assertFalse(mt.done) + self.assertRaises(zmq.NotDone, mt.wait, 0.1) + del m + time.sleep(0.1) + self.assertRaises(zmq.NotDone, mt.wait, 0.1) + self.assertFalse(mt.done) + del m2 + self.assertTrue(mt.wait() is None) + self.assertTrue(mt.done) + + + def test_buffer_in(self): + """test using a buffer as input""" + ins = b("§§¶•ªº˜µ¬˚…∆˙åß∂©œ∑´†≈ç√") + m = zmq.Frame(view(ins)) + + def test_bad_buffer_in(self): + """test using a bad object""" + self.assertRaises(TypeError, zmq.Frame, 5) + self.assertRaises(TypeError, zmq.Frame, object()) + + def test_buffer_out(self): + """receiving buffered output""" + ins = b("§§¶•ªº˜µ¬˚…∆˙åß∂©œ∑´†≈ç√") + m = zmq.Frame(ins) + outb = m.buffer + self.assertTrue(isinstance(outb, view)) + self.assert_(outb is m.buffer) + self.assert_(m.buffer is m.buffer) + + def test_multisend(self): + """ensure that a message remains intact after multiple sends""" + a,b = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + s = b"message" + m = zmq.Frame(s) + self.assertEqual(s, m.bytes) + + a.send(m, copy=False) + time.sleep(0.1) + self.assertEqual(s, m.bytes) + a.send(m, copy=False) + time.sleep(0.1) + self.assertEqual(s, m.bytes) + a.send(m, copy=True) + time.sleep(0.1) + self.assertEqual(s, m.bytes) + a.send(m, copy=True) + time.sleep(0.1) + self.assertEqual(s, m.bytes) + for i in range(4): + r = b.recv() + self.assertEqual(s,r) + self.assertEqual(s, m.bytes) + + def test_buffer_numpy(self): + """test non-copying numpy array messages""" + try: + import numpy + except ImportError: + raise SkipTest("numpy required") + rand = numpy.random.randint + shapes = [ rand(2,16) for i in range(5) ] + for i in range(1,len(shapes)+1): + shape = shapes[:i] + A = numpy.random.random(shape) + m = zmq.Frame(A) + if view.__name__ == 'buffer': + self.assertEqual(A.data, m.buffer) + B = numpy.frombuffer(m.buffer,dtype=A.dtype).reshape(A.shape) + else: + self.assertEqual(memoryview(A), m.buffer) + B = numpy.array(m.buffer,dtype=A.dtype).reshape(A.shape) + self.assertEqual((A==B).all(), True) + + def test_memoryview(self): + """test messages from memoryview""" + major,minor = sys.version_info[:2] + if not (major >= 3 or (major == 2 and minor >= 7)): + raise SkipTest("memoryviews only in python >= 2.7") + + s = b'carrotjuice' + v = memoryview(s) + m = zmq.Frame(s) + buf = m.buffer + s2 = buf.tobytes() + self.assertEqual(s2,s) + self.assertEqual(m.bytes,s) + + def test_noncopying_recv(self): + """check for clobbering message buffers""" + null = b'\0'*64 + sa,sb = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + for i in range(32): + # try a few times + sb.send(null, copy=False) + m = sa.recv(copy=False) + mb = m.bytes + # buf = view(m) + buf = m.buffer + del m + for i in range(5): + ff=b'\xff'*(40 + i*10) + sb.send(ff, copy=False) + m2 = sa.recv(copy=False) + if view.__name__ == 'buffer': + b = bytes(buf) + else: + b = buf.tobytes() + self.assertEqual(b, null) + self.assertEqual(mb, null) + self.assertEqual(m2.bytes, ff) + + @skip_pypy + def test_buffer_numpy(self): + """test non-copying numpy array messages""" + try: + import numpy + except ImportError: + raise SkipTest("requires numpy") + if sys.version_info < (2,7): + raise SkipTest("requires new-style buffer interface (py >= 2.7)") + rand = numpy.random.randint + shapes = [ rand(2,5) for i in range(5) ] + a,b = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + dtypes = [int, float, '>i4', 'B'] + for i in range(1,len(shapes)+1): + shape = shapes[:i] + for dt in dtypes: + A = numpy.empty(shape, dtype=dt) + while numpy.isnan(A).any(): + # don't let nan sneak in + A = numpy.ndarray(shape, dtype=dt) + a.send(A, copy=False) + msg = b.recv(copy=False) + + B = numpy.frombuffer(msg, A.dtype).reshape(A.shape) + self.assertEqual(A.shape, B.shape) + self.assertTrue((A==B).all()) + A = numpy.empty(shape, dtype=[('a', int), ('b', float), ('c', 'a32')]) + A['a'] = 1024 + A['b'] = 1e9 + A['c'] = 'hello there' + a.send(A, copy=False) + msg = b.recv(copy=False) + + B = numpy.frombuffer(msg, A.dtype).reshape(A.shape) + self.assertEqual(A.shape, B.shape) + self.assertTrue((A==B).all()) + + def test_frame_more(self): + """test Frame.more attribute""" + frame = zmq.Frame(b"hello") + self.assertFalse(frame.more) + sa,sb = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + sa.send_multipart([b'hi', b'there']) + frame = self.recv(sb, copy=False) + self.assertTrue(frame.more) + if zmq.zmq_version_info()[0] >= 3 and not PYPY: + self.assertTrue(frame.get(zmq.MORE)) + frame = self.recv(sb, copy=False) + self.assertFalse(frame.more) + if zmq.zmq_version_info()[0] >= 3 and not PYPY: + self.assertFalse(frame.get(zmq.MORE)) + diff --git a/scripts/external_libs/zmq/tests/test_monitor.py b/scripts/external_libs/zmq/tests/test_monitor.py new file mode 100644 index 00000000..4f035388 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_monitor.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import sys +import time +import struct + +from unittest import TestCase + +import zmq +from zmq.tests import BaseZMQTestCase, skip_if, skip_pypy +from zmq.utils.monitor import recv_monitor_message + +skip_lt_4 = skip_if(zmq.zmq_version_info() < (4,), "requires zmq >= 4") + +class TestSocketMonitor(BaseZMQTestCase): + + @skip_lt_4 + def test_monitor(self): + """Test monitoring interface for sockets.""" + s_rep = self.context.socket(zmq.REP) + s_req = self.context.socket(zmq.REQ) + self.sockets.extend([s_rep, s_req]) + s_req.bind("tcp://127.0.0.1:6666") + # try monitoring the REP socket + + s_rep.monitor("inproc://monitor.rep", zmq.EVENT_ALL) + # create listening socket for monitor + s_event = self.context.socket(zmq.PAIR) + self.sockets.append(s_event) + s_event.connect("inproc://monitor.rep") + s_event.linger = 0 + # test receive event for connect event + s_rep.connect("tcp://127.0.0.1:6666") + m = recv_monitor_message(s_event) + if m['event'] == zmq.EVENT_CONNECT_DELAYED: + self.assertEqual(m['endpoint'], b"tcp://127.0.0.1:6666") + # test receive event for connected event + m = recv_monitor_message(s_event) + self.assertEqual(m['event'], zmq.EVENT_CONNECTED) + self.assertEqual(m['endpoint'], b"tcp://127.0.0.1:6666") + + # test monitor can be disabled. + s_rep.disable_monitor() + m = recv_monitor_message(s_event) + self.assertEqual(m['event'], zmq.EVENT_MONITOR_STOPPED) + + + @skip_lt_4 + def test_monitor_connected(self): + """Test connected monitoring socket.""" + s_rep = self.context.socket(zmq.REP) + s_req = self.context.socket(zmq.REQ) + self.sockets.extend([s_rep, s_req]) + s_req.bind("tcp://127.0.0.1:6667") + # try monitoring the REP socket + # create listening socket for monitor + s_event = s_rep.get_monitor_socket() + s_event.linger = 0 + self.sockets.append(s_event) + # test receive event for connect event + s_rep.connect("tcp://127.0.0.1:6667") + m = recv_monitor_message(s_event) + if m['event'] == zmq.EVENT_CONNECT_DELAYED: + self.assertEqual(m['endpoint'], b"tcp://127.0.0.1:6667") + # test receive event for connected event + m = recv_monitor_message(s_event) + self.assertEqual(m['event'], zmq.EVENT_CONNECTED) + self.assertEqual(m['endpoint'], b"tcp://127.0.0.1:6667") diff --git a/scripts/external_libs/zmq/tests/test_monqueue.py b/scripts/external_libs/zmq/tests/test_monqueue.py new file mode 100644 index 00000000..e855602e --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_monqueue.py @@ -0,0 +1,227 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import time +from unittest import TestCase + +import zmq +from zmq import devices + +from zmq.tests import BaseZMQTestCase, SkipTest, PYPY +from zmq.utils.strtypes import unicode + + +if PYPY or zmq.zmq_version_info() >= (4,1): + # cleanup of shared Context doesn't work on PyPy + # there also seems to be a bug in cleanup in libzmq-4.1 (zeromq/libzmq#1052) + devices.Device.context_factory = zmq.Context + + +class TestMonitoredQueue(BaseZMQTestCase): + + sockets = [] + + def build_device(self, mon_sub=b"", in_prefix=b'in', out_prefix=b'out'): + self.device = devices.ThreadMonitoredQueue(zmq.PAIR, zmq.PAIR, zmq.PUB, + in_prefix, out_prefix) + alice = self.context.socket(zmq.PAIR) + bob = self.context.socket(zmq.PAIR) + mon = self.context.socket(zmq.SUB) + + aport = alice.bind_to_random_port('tcp://127.0.0.1') + bport = bob.bind_to_random_port('tcp://127.0.0.1') + mport = mon.bind_to_random_port('tcp://127.0.0.1') + mon.setsockopt(zmq.SUBSCRIBE, mon_sub) + + self.device.connect_in("tcp://127.0.0.1:%i"%aport) + self.device.connect_out("tcp://127.0.0.1:%i"%bport) + self.device.connect_mon("tcp://127.0.0.1:%i"%mport) + self.device.start() + time.sleep(.2) + try: + # this is currenlty necessary to ensure no dropped monitor messages + # see LIBZMQ-248 for more info + mon.recv_multipart(zmq.NOBLOCK) + except zmq.ZMQError: + pass + self.sockets.extend([alice, bob, mon]) + return alice, bob, mon + + + def teardown_device(self): + for socket in self.sockets: + socket.close() + del socket + del self.device + + def test_reply(self): + alice, bob, mon = self.build_device() + alices = b"hello bob".split() + alice.send_multipart(alices) + bobs = self.recv_multipart(bob) + self.assertEqual(alices, bobs) + bobs = b"hello alice".split() + bob.send_multipart(bobs) + alices = self.recv_multipart(alice) + self.assertEqual(alices, bobs) + self.teardown_device() + + def test_queue(self): + alice, bob, mon = self.build_device() + alices = b"hello bob".split() + alice.send_multipart(alices) + alices2 = b"hello again".split() + alice.send_multipart(alices2) + alices3 = b"hello again and again".split() + alice.send_multipart(alices3) + bobs = self.recv_multipart(bob) + self.assertEqual(alices, bobs) + bobs = self.recv_multipart(bob) + self.assertEqual(alices2, bobs) + bobs = self.recv_multipart(bob) + self.assertEqual(alices3, bobs) + bobs = b"hello alice".split() + bob.send_multipart(bobs) + alices = self.recv_multipart(alice) + self.assertEqual(alices, bobs) + self.teardown_device() + + def test_monitor(self): + alice, bob, mon = self.build_device() + alices = b"hello bob".split() + alice.send_multipart(alices) + alices2 = b"hello again".split() + alice.send_multipart(alices2) + alices3 = b"hello again and again".split() + alice.send_multipart(alices3) + bobs = self.recv_multipart(bob) + self.assertEqual(alices, bobs) + mons = self.recv_multipart(mon) + self.assertEqual([b'in']+bobs, mons) + bobs = self.recv_multipart(bob) + self.assertEqual(alices2, bobs) + bobs = self.recv_multipart(bob) + self.assertEqual(alices3, bobs) + mons = self.recv_multipart(mon) + self.assertEqual([b'in']+alices2, mons) + bobs = b"hello alice".split() + bob.send_multipart(bobs) + alices = self.recv_multipart(alice) + self.assertEqual(alices, bobs) + mons = self.recv_multipart(mon) + self.assertEqual([b'in']+alices3, mons) + mons = self.recv_multipart(mon) + self.assertEqual([b'out']+bobs, mons) + self.teardown_device() + + def test_prefix(self): + alice, bob, mon = self.build_device(b"", b'foo', b'bar') + alices = b"hello bob".split() + alice.send_multipart(alices) + alices2 = b"hello again".split() + alice.send_multipart(alices2) + alices3 = b"hello again and again".split() + alice.send_multipart(alices3) + bobs = self.recv_multipart(bob) + self.assertEqual(alices, bobs) + mons = self.recv_multipart(mon) + self.assertEqual([b'foo']+bobs, mons) + bobs = self.recv_multipart(bob) + self.assertEqual(alices2, bobs) + bobs = self.recv_multipart(bob) + self.assertEqual(alices3, bobs) + mons = self.recv_multipart(mon) + self.assertEqual([b'foo']+alices2, mons) + bobs = b"hello alice".split() + bob.send_multipart(bobs) + alices = self.recv_multipart(alice) + self.assertEqual(alices, bobs) + mons = self.recv_multipart(mon) + self.assertEqual([b'foo']+alices3, mons) + mons = self.recv_multipart(mon) + self.assertEqual([b'bar']+bobs, mons) + self.teardown_device() + + def test_monitor_subscribe(self): + alice, bob, mon = self.build_device(b"out") + alices = b"hello bob".split() + alice.send_multipart(alices) + alices2 = b"hello again".split() + alice.send_multipart(alices2) + alices3 = b"hello again and again".split() + alice.send_multipart(alices3) + bobs = self.recv_multipart(bob) + self.assertEqual(alices, bobs) + bobs = self.recv_multipart(bob) + self.assertEqual(alices2, bobs) + bobs = self.recv_multipart(bob) + self.assertEqual(alices3, bobs) + bobs = b"hello alice".split() + bob.send_multipart(bobs) + alices = self.recv_multipart(alice) + self.assertEqual(alices, bobs) + mons = self.recv_multipart(mon) + self.assertEqual([b'out']+bobs, mons) + self.teardown_device() + + def test_router_router(self): + """test router-router MQ devices""" + dev = devices.ThreadMonitoredQueue(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') + self.device = dev + dev.setsockopt_in(zmq.LINGER, 0) + dev.setsockopt_out(zmq.LINGER, 0) + dev.setsockopt_mon(zmq.LINGER, 0) + + binder = self.context.socket(zmq.DEALER) + porta = binder.bind_to_random_port('tcp://127.0.0.1') + portb = binder.bind_to_random_port('tcp://127.0.0.1') + binder.close() + time.sleep(0.1) + a = self.context.socket(zmq.DEALER) + a.identity = b'a' + b = self.context.socket(zmq.DEALER) + b.identity = b'b' + self.sockets.extend([a, b]) + + a.connect('tcp://127.0.0.1:%i'%porta) + dev.bind_in('tcp://127.0.0.1:%i'%porta) + b.connect('tcp://127.0.0.1:%i'%portb) + dev.bind_out('tcp://127.0.0.1:%i'%portb) + dev.start() + time.sleep(0.2) + if zmq.zmq_version_info() >= (3,1,0): + # flush erroneous poll state, due to LIBZMQ-280 + ping_msg = [ b'ping', b'pong' ] + for s in (a,b): + s.send_multipart(ping_msg) + try: + s.recv(zmq.NOBLOCK) + except zmq.ZMQError: + pass + msg = [ b'hello', b'there' ] + a.send_multipart([b'b']+msg) + bmsg = self.recv_multipart(b) + self.assertEqual(bmsg, [b'a']+msg) + b.send_multipart(bmsg) + amsg = self.recv_multipart(a) + self.assertEqual(amsg, [b'b']+msg) + self.teardown_device() + + def test_default_mq_args(self): + self.device = dev = devices.ThreadMonitoredQueue(zmq.ROUTER, zmq.DEALER, zmq.PUB) + dev.setsockopt_in(zmq.LINGER, 0) + dev.setsockopt_out(zmq.LINGER, 0) + dev.setsockopt_mon(zmq.LINGER, 0) + # this will raise if default args are wrong + dev.start() + self.teardown_device() + + def test_mq_check_prefix(self): + ins = self.context.socket(zmq.ROUTER) + outs = self.context.socket(zmq.DEALER) + mons = self.context.socket(zmq.PUB) + self.sockets.extend([ins, outs, mons]) + + ins = unicode('in') + outs = unicode('out') + self.assertRaises(TypeError, devices.monitoredqueue, ins, outs, mons) diff --git a/scripts/external_libs/zmq/tests/test_multipart.py b/scripts/external_libs/zmq/tests/test_multipart.py new file mode 100644 index 00000000..24d41be0 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_multipart.py @@ -0,0 +1,35 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import zmq + + +from zmq.tests import BaseZMQTestCase, SkipTest, have_gevent, GreenTest + + +class TestMultipart(BaseZMQTestCase): + + def test_router_dealer(self): + router, dealer = self.create_bound_pair(zmq.ROUTER, zmq.DEALER) + + msg1 = b'message1' + dealer.send(msg1) + ident = self.recv(router) + more = router.rcvmore + self.assertEqual(more, True) + msg2 = self.recv(router) + self.assertEqual(msg1, msg2) + more = router.rcvmore + self.assertEqual(more, False) + + def test_basic_multipart(self): + a,b = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + msg = [ b'hi', b'there', b'b'] + a.send_multipart(msg) + recvd = b.recv_multipart() + self.assertEqual(msg, recvd) + +if have_gevent: + class TestMultipartGreen(GreenTest, TestMultipart): + pass diff --git a/scripts/external_libs/zmq/tests/test_pair.py b/scripts/external_libs/zmq/tests/test_pair.py new file mode 100644 index 00000000..e88c1e8b --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_pair.py @@ -0,0 +1,53 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import zmq + + +from zmq.tests import BaseZMQTestCase, have_gevent, GreenTest + + +x = b' ' +class TestPair(BaseZMQTestCase): + + def test_basic(self): + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + + msg1 = b'message1' + msg2 = self.ping_pong(s1, s2, msg1) + self.assertEqual(msg1, msg2) + + def test_multiple(self): + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + + for i in range(10): + msg = i*x + s1.send(msg) + + for i in range(10): + msg = i*x + s2.send(msg) + + for i in range(10): + msg = s1.recv() + self.assertEqual(msg, i*x) + + for i in range(10): + msg = s2.recv() + self.assertEqual(msg, i*x) + + def test_json(self): + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + o = dict(a=10,b=list(range(10))) + o2 = self.ping_pong_json(s1, s2, o) + + def test_pyobj(self): + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + o = dict(a=10,b=range(10)) + o2 = self.ping_pong_pyobj(s1, s2, o) + +if have_gevent: + class TestReqRepGreen(GreenTest, TestPair): + pass + diff --git a/scripts/external_libs/zmq/tests/test_poll.py b/scripts/external_libs/zmq/tests/test_poll.py new file mode 100644 index 00000000..57346c89 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_poll.py @@ -0,0 +1,229 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import time +from unittest import TestCase + +import zmq + +from zmq.tests import PollZMQTestCase, have_gevent, GreenTest + +def wait(): + time.sleep(.25) + + +class TestPoll(PollZMQTestCase): + + Poller = zmq.Poller + + # This test is failing due to this issue: + # http://github.com/sustrik/zeromq2/issues#issue/26 + def test_pair(self): + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + + # Sleep to allow sockets to connect. + wait() + + poller = self.Poller() + poller.register(s1, zmq.POLLIN|zmq.POLLOUT) + poller.register(s2, zmq.POLLIN|zmq.POLLOUT) + # Poll result should contain both sockets + socks = dict(poller.poll()) + # Now make sure that both are send ready. + self.assertEqual(socks[s1], zmq.POLLOUT) + self.assertEqual(socks[s2], zmq.POLLOUT) + # Now do a send on both, wait and test for zmq.POLLOUT|zmq.POLLIN + s1.send(b'msg1') + s2.send(b'msg2') + wait() + socks = dict(poller.poll()) + self.assertEqual(socks[s1], zmq.POLLOUT|zmq.POLLIN) + self.assertEqual(socks[s2], zmq.POLLOUT|zmq.POLLIN) + # Make sure that both are in POLLOUT after recv. + s1.recv() + s2.recv() + socks = dict(poller.poll()) + self.assertEqual(socks[s1], zmq.POLLOUT) + self.assertEqual(socks[s2], zmq.POLLOUT) + + poller.unregister(s1) + poller.unregister(s2) + + # Wait for everything to finish. + wait() + + def test_reqrep(self): + s1, s2 = self.create_bound_pair(zmq.REP, zmq.REQ) + + # Sleep to allow sockets to connect. + wait() + + poller = self.Poller() + poller.register(s1, zmq.POLLIN|zmq.POLLOUT) + poller.register(s2, zmq.POLLIN|zmq.POLLOUT) + + # Make sure that s1 is in state 0 and s2 is in POLLOUT + socks = dict(poller.poll()) + self.assertEqual(s1 in socks, 0) + self.assertEqual(socks[s2], zmq.POLLOUT) + + # Make sure that s2 goes immediately into state 0 after send. + s2.send(b'msg1') + socks = dict(poller.poll()) + self.assertEqual(s2 in socks, 0) + + # Make sure that s1 goes into POLLIN state after a time.sleep(). + time.sleep(0.5) + socks = dict(poller.poll()) + self.assertEqual(socks[s1], zmq.POLLIN) + + # Make sure that s1 goes into POLLOUT after recv. + s1.recv() + socks = dict(poller.poll()) + self.assertEqual(socks[s1], zmq.POLLOUT) + + # Make sure s1 goes into state 0 after send. + s1.send(b'msg2') + socks = dict(poller.poll()) + self.assertEqual(s1 in socks, 0) + + # Wait and then see that s2 is in POLLIN. + time.sleep(0.5) + socks = dict(poller.poll()) + self.assertEqual(socks[s2], zmq.POLLIN) + + # Make sure that s2 is in POLLOUT after recv. + s2.recv() + socks = dict(poller.poll()) + self.assertEqual(socks[s2], zmq.POLLOUT) + + poller.unregister(s1) + poller.unregister(s2) + + # Wait for everything to finish. + wait() + + def test_no_events(self): + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + poller = self.Poller() + poller.register(s1, zmq.POLLIN|zmq.POLLOUT) + poller.register(s2, 0) + self.assertTrue(s1 in poller) + self.assertFalse(s2 in poller) + poller.register(s1, 0) + self.assertFalse(s1 in poller) + + def test_pubsub(self): + s1, s2 = self.create_bound_pair(zmq.PUB, zmq.SUB) + s2.setsockopt(zmq.SUBSCRIBE, b'') + + # Sleep to allow sockets to connect. + wait() + + poller = self.Poller() + poller.register(s1, zmq.POLLIN|zmq.POLLOUT) + poller.register(s2, zmq.POLLIN) + + # Now make sure that both are send ready. + socks = dict(poller.poll()) + self.assertEqual(socks[s1], zmq.POLLOUT) + self.assertEqual(s2 in socks, 0) + # Make sure that s1 stays in POLLOUT after a send. + s1.send(b'msg1') + socks = dict(poller.poll()) + self.assertEqual(socks[s1], zmq.POLLOUT) + + # Make sure that s2 is POLLIN after waiting. + wait() + socks = dict(poller.poll()) + self.assertEqual(socks[s2], zmq.POLLIN) + + # Make sure that s2 goes into 0 after recv. + s2.recv() + socks = dict(poller.poll()) + self.assertEqual(s2 in socks, 0) + + poller.unregister(s1) + poller.unregister(s2) + + # Wait for everything to finish. + wait() + def test_timeout(self): + """make sure Poller.poll timeout has the right units (milliseconds).""" + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + poller = self.Poller() + poller.register(s1, zmq.POLLIN) + tic = time.time() + evt = poller.poll(.005) + toc = time.time() + self.assertTrue(toc-tic < 0.1) + tic = time.time() + evt = poller.poll(5) + toc = time.time() + self.assertTrue(toc-tic < 0.1) + self.assertTrue(toc-tic > .001) + tic = time.time() + evt = poller.poll(500) + toc = time.time() + self.assertTrue(toc-tic < 1) + self.assertTrue(toc-tic > 0.1) + +class TestSelect(PollZMQTestCase): + + def test_pair(self): + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + + # Sleep to allow sockets to connect. + wait() + + rlist, wlist, xlist = zmq.select([s1, s2], [s1, s2], [s1, s2]) + self.assert_(s1 in wlist) + self.assert_(s2 in wlist) + self.assert_(s1 not in rlist) + self.assert_(s2 not in rlist) + + def test_timeout(self): + """make sure select timeout has the right units (seconds).""" + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + tic = time.time() + r,w,x = zmq.select([s1,s2],[],[],.005) + toc = time.time() + self.assertTrue(toc-tic < 1) + self.assertTrue(toc-tic > 0.001) + tic = time.time() + r,w,x = zmq.select([s1,s2],[],[],.25) + toc = time.time() + self.assertTrue(toc-tic < 1) + self.assertTrue(toc-tic > 0.1) + + +if have_gevent: + import gevent + from zmq import green as gzmq + + class TestPollGreen(GreenTest, TestPoll): + Poller = gzmq.Poller + + def test_wakeup(self): + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + poller = self.Poller() + poller.register(s2, zmq.POLLIN) + + tic = time.time() + r = gevent.spawn(lambda: poller.poll(10000)) + s = gevent.spawn(lambda: s1.send(b'msg1')) + r.join() + toc = time.time() + self.assertTrue(toc-tic < 1) + + def test_socket_poll(self): + s1, s2 = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + + tic = time.time() + r = gevent.spawn(lambda: s2.poll(10000)) + s = gevent.spawn(lambda: s1.send(b'msg1')) + r.join() + toc = time.time() + self.assertTrue(toc-tic < 1) + diff --git a/scripts/external_libs/zmq/tests/test_pubsub.py b/scripts/external_libs/zmq/tests/test_pubsub.py new file mode 100644 index 00000000..a3ee22aa --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_pubsub.py @@ -0,0 +1,41 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import time +from unittest import TestCase + +import zmq + +from zmq.tests import BaseZMQTestCase, have_gevent, GreenTest + + +class TestPubSub(BaseZMQTestCase): + + pass + + # We are disabling this test while an issue is being resolved. + def test_basic(self): + s1, s2 = self.create_bound_pair(zmq.PUB, zmq.SUB) + s2.setsockopt(zmq.SUBSCRIBE,b'') + time.sleep(0.1) + msg1 = b'message' + s1.send(msg1) + msg2 = s2.recv() # This is blocking! + self.assertEqual(msg1, msg2) + + def test_topic(self): + s1, s2 = self.create_bound_pair(zmq.PUB, zmq.SUB) + s2.setsockopt(zmq.SUBSCRIBE, b'x') + time.sleep(0.1) + msg1 = b'message' + s1.send(msg1) + self.assertRaisesErrno(zmq.EAGAIN, s2.recv, zmq.NOBLOCK) + msg1 = b'xmessage' + s1.send(msg1) + msg2 = s2.recv() + self.assertEqual(msg1, msg2) + +if have_gevent: + class TestPubSubGreen(GreenTest, TestPubSub): + pass diff --git a/scripts/external_libs/zmq/tests/test_reqrep.py b/scripts/external_libs/zmq/tests/test_reqrep.py new file mode 100644 index 00000000..de17f2b3 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_reqrep.py @@ -0,0 +1,62 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +from unittest import TestCase + +import zmq +from zmq.tests import BaseZMQTestCase, have_gevent, GreenTest + + +class TestReqRep(BaseZMQTestCase): + + def test_basic(self): + s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP) + + msg1 = b'message 1' + msg2 = self.ping_pong(s1, s2, msg1) + self.assertEqual(msg1, msg2) + + def test_multiple(self): + s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP) + + for i in range(10): + msg1 = i*b' ' + msg2 = self.ping_pong(s1, s2, msg1) + self.assertEqual(msg1, msg2) + + def test_bad_send_recv(self): + s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP) + + if zmq.zmq_version() != '2.1.8': + # this doesn't work on 2.1.8 + for copy in (True,False): + self.assertRaisesErrno(zmq.EFSM, s1.recv, copy=copy) + self.assertRaisesErrno(zmq.EFSM, s2.send, b'asdf', copy=copy) + + # I have to have this or we die on an Abort trap. + msg1 = b'asdf' + msg2 = self.ping_pong(s1, s2, msg1) + self.assertEqual(msg1, msg2) + + def test_json(self): + s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP) + o = dict(a=10,b=list(range(10))) + o2 = self.ping_pong_json(s1, s2, o) + + def test_pyobj(self): + s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP) + o = dict(a=10,b=range(10)) + o2 = self.ping_pong_pyobj(s1, s2, o) + + def test_large_msg(self): + s1, s2 = self.create_bound_pair(zmq.REQ, zmq.REP) + msg1 = 10000*b'X' + + for i in range(10): + msg2 = self.ping_pong(s1, s2, msg1) + self.assertEqual(msg1, msg2) + +if have_gevent: + class TestReqRepGreen(GreenTest, TestReqRep): + pass diff --git a/scripts/external_libs/zmq/tests/test_security.py b/scripts/external_libs/zmq/tests/test_security.py new file mode 100644 index 00000000..687b7e0f --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_security.py @@ -0,0 +1,212 @@ +"""Test libzmq security (libzmq >= 3.3.0)""" +# -*- coding: utf8 -*- + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import os +from threading import Thread + +import zmq +from zmq.tests import ( + BaseZMQTestCase, SkipTest, PYPY +) +from zmq.utils import z85 + + +USER = b"admin" +PASS = b"password" + +class TestSecurity(BaseZMQTestCase): + + def setUp(self): + if zmq.zmq_version_info() < (4,0): + raise SkipTest("security is new in libzmq 4.0") + try: + zmq.curve_keypair() + except zmq.ZMQError: + raise SkipTest("security requires libzmq to be linked against libsodium") + super(TestSecurity, self).setUp() + + + def zap_handler(self): + socket = self.context.socket(zmq.REP) + socket.bind("inproc://zeromq.zap.01") + try: + msg = self.recv_multipart(socket) + + version, sequence, domain, address, identity, mechanism = msg[:6] + if mechanism == b'PLAIN': + username, password = msg[6:] + elif mechanism == b'CURVE': + key = msg[6] + + self.assertEqual(version, b"1.0") + self.assertEqual(identity, b"IDENT") + reply = [version, sequence] + if mechanism == b'CURVE' or \ + (mechanism == b'PLAIN' and username == USER and password == PASS) or \ + (mechanism == b'NULL'): + reply.extend([ + b"200", + b"OK", + b"anonymous", + b"\5Hello\0\0\0\5World", + ]) + else: + reply.extend([ + b"400", + b"Invalid username or password", + b"", + b"", + ]) + socket.send_multipart(reply) + finally: + socket.close() + + def start_zap(self): + self.zap_thread = Thread(target=self.zap_handler) + self.zap_thread.start() + + def stop_zap(self): + self.zap_thread.join() + + def bounce(self, server, client, test_metadata=True): + msg = [os.urandom(64), os.urandom(64)] + client.send_multipart(msg) + frames = self.recv_multipart(server, copy=False) + recvd = list(map(lambda x: x.bytes, frames)) + + try: + if test_metadata and not PYPY: + for frame in frames: + self.assertEqual(frame.get('User-Id'), 'anonymous') + self.assertEqual(frame.get('Hello'), 'World') + self.assertEqual(frame['Socket-Type'], 'DEALER') + except zmq.ZMQVersionError: + pass + + self.assertEqual(recvd, msg) + server.send_multipart(recvd) + msg2 = self.recv_multipart(client) + self.assertEqual(msg2, msg) + + def test_null(self): + """test NULL (default) security""" + server = self.socket(zmq.DEALER) + client = self.socket(zmq.DEALER) + self.assertEqual(client.MECHANISM, zmq.NULL) + self.assertEqual(server.mechanism, zmq.NULL) + self.assertEqual(client.plain_server, 0) + self.assertEqual(server.plain_server, 0) + iface = 'tcp://127.0.0.1' + port = server.bind_to_random_port(iface) + client.connect("%s:%i" % (iface, port)) + self.bounce(server, client, False) + + def test_plain(self): + """test PLAIN authentication""" + server = self.socket(zmq.DEALER) + server.identity = b'IDENT' + client = self.socket(zmq.DEALER) + self.assertEqual(client.plain_username, b'') + self.assertEqual(client.plain_password, b'') + client.plain_username = USER + client.plain_password = PASS + self.assertEqual(client.getsockopt(zmq.PLAIN_USERNAME), USER) + self.assertEqual(client.getsockopt(zmq.PLAIN_PASSWORD), PASS) + self.assertEqual(client.plain_server, 0) + self.assertEqual(server.plain_server, 0) + server.plain_server = True + self.assertEqual(server.mechanism, zmq.PLAIN) + self.assertEqual(client.mechanism, zmq.PLAIN) + + assert not client.plain_server + assert server.plain_server + + self.start_zap() + + iface = 'tcp://127.0.0.1' + port = server.bind_to_random_port(iface) + client.connect("%s:%i" % (iface, port)) + self.bounce(server, client) + self.stop_zap() + + def skip_plain_inauth(self): + """test PLAIN failed authentication""" + server = self.socket(zmq.DEALER) + server.identity = b'IDENT' + client = self.socket(zmq.DEALER) + self.sockets.extend([server, client]) + client.plain_username = USER + client.plain_password = b'incorrect' + server.plain_server = True + self.assertEqual(server.mechanism, zmq.PLAIN) + self.assertEqual(client.mechanism, zmq.PLAIN) + + self.start_zap() + + iface = 'tcp://127.0.0.1' + port = server.bind_to_random_port(iface) + client.connect("%s:%i" % (iface, port)) + client.send(b'ping') + server.rcvtimeo = 250 + self.assertRaisesErrno(zmq.EAGAIN, server.recv) + self.stop_zap() + + def test_keypair(self): + """test curve_keypair""" + try: + public, secret = zmq.curve_keypair() + except zmq.ZMQError: + raise SkipTest("CURVE unsupported") + + self.assertEqual(type(secret), bytes) + self.assertEqual(type(public), bytes) + self.assertEqual(len(secret), 40) + self.assertEqual(len(public), 40) + + # verify that it is indeed Z85 + bsecret, bpublic = [ z85.decode(key) for key in (public, secret) ] + self.assertEqual(type(bsecret), bytes) + self.assertEqual(type(bpublic), bytes) + self.assertEqual(len(bsecret), 32) + self.assertEqual(len(bpublic), 32) + + + def test_curve(self): + """test CURVE encryption""" + server = self.socket(zmq.DEALER) + server.identity = b'IDENT' + client = self.socket(zmq.DEALER) + self.sockets.extend([server, client]) + try: + server.curve_server = True + except zmq.ZMQError as e: + # will raise EINVAL if not linked against libsodium + if e.errno == zmq.EINVAL: + raise SkipTest("CURVE unsupported") + + server_public, server_secret = zmq.curve_keypair() + client_public, client_secret = zmq.curve_keypair() + + server.curve_secretkey = server_secret + server.curve_publickey = server_public + client.curve_serverkey = server_public + client.curve_publickey = client_public + client.curve_secretkey = client_secret + + self.assertEqual(server.mechanism, zmq.CURVE) + self.assertEqual(client.mechanism, zmq.CURVE) + + self.assertEqual(server.get(zmq.CURVE_SERVER), True) + self.assertEqual(client.get(zmq.CURVE_SERVER), False) + + self.start_zap() + + iface = 'tcp://127.0.0.1' + port = server.bind_to_random_port(iface) + client.connect("%s:%i" % (iface, port)) + self.bounce(server, client) + self.stop_zap() + diff --git a/scripts/external_libs/zmq/tests/test_socket.py b/scripts/external_libs/zmq/tests/test_socket.py new file mode 100644 index 00000000..5c842edc --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_socket.py @@ -0,0 +1,450 @@ +# -*- coding: utf8 -*- +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import time +import warnings + +import zmq +from zmq.tests import ( + BaseZMQTestCase, SkipTest, have_gevent, GreenTest, skip_pypy, skip_if +) +from zmq.utils.strtypes import bytes, unicode + + +class TestSocket(BaseZMQTestCase): + + def test_create(self): + ctx = self.Context() + s = ctx.socket(zmq.PUB) + # Superluminal protocol not yet implemented + self.assertRaisesErrno(zmq.EPROTONOSUPPORT, s.bind, 'ftl://a') + self.assertRaisesErrno(zmq.EPROTONOSUPPORT, s.connect, 'ftl://a') + self.assertRaisesErrno(zmq.EINVAL, s.bind, 'tcp://') + s.close() + del ctx + + def test_context_manager(self): + url = 'inproc://a' + with self.Context() as ctx: + with ctx.socket(zmq.PUSH) as a: + a.bind(url) + with ctx.socket(zmq.PULL) as b: + b.connect(url) + msg = b'hi' + a.send(msg) + rcvd = self.recv(b) + self.assertEqual(rcvd, msg) + self.assertEqual(b.closed, True) + self.assertEqual(a.closed, True) + self.assertEqual(ctx.closed, True) + + def test_dir(self): + ctx = self.Context() + s = ctx.socket(zmq.PUB) + self.assertTrue('send' in dir(s)) + self.assertTrue('IDENTITY' in dir(s)) + self.assertTrue('AFFINITY' in dir(s)) + self.assertTrue('FD' in dir(s)) + s.close() + ctx.term() + + def test_bind_unicode(self): + s = self.socket(zmq.PUB) + p = s.bind_to_random_port(unicode("tcp://*")) + + def test_connect_unicode(self): + s = self.socket(zmq.PUB) + s.connect(unicode("tcp://127.0.0.1:5555")) + + def test_bind_to_random_port(self): + # Check that bind_to_random_port do not hide usefull exception + ctx = self.Context() + c = ctx.socket(zmq.PUB) + # Invalid format + try: + c.bind_to_random_port('tcp:*') + except zmq.ZMQError as e: + self.assertEqual(e.errno, zmq.EINVAL) + # Invalid protocol + try: + c.bind_to_random_port('rand://*') + except zmq.ZMQError as e: + self.assertEqual(e.errno, zmq.EPROTONOSUPPORT) + + def test_identity(self): + s = self.context.socket(zmq.PULL) + self.sockets.append(s) + ident = b'identity\0\0' + s.identity = ident + self.assertEqual(s.get(zmq.IDENTITY), ident) + + def test_unicode_sockopts(self): + """test setting/getting sockopts with unicode strings""" + topic = "tést" + if str is not unicode: + topic = topic.decode('utf8') + p,s = self.create_bound_pair(zmq.PUB, zmq.SUB) + self.assertEqual(s.send_unicode, s.send_unicode) + self.assertEqual(p.recv_unicode, p.recv_unicode) + self.assertRaises(TypeError, s.setsockopt, zmq.SUBSCRIBE, topic) + self.assertRaises(TypeError, s.setsockopt, zmq.IDENTITY, topic) + s.setsockopt_unicode(zmq.IDENTITY, topic, 'utf16') + self.assertRaises(TypeError, s.setsockopt, zmq.AFFINITY, topic) + s.setsockopt_unicode(zmq.SUBSCRIBE, topic) + self.assertRaises(TypeError, s.getsockopt_unicode, zmq.AFFINITY) + self.assertRaisesErrno(zmq.EINVAL, s.getsockopt_unicode, zmq.SUBSCRIBE) + + identb = s.getsockopt(zmq.IDENTITY) + identu = identb.decode('utf16') + identu2 = s.getsockopt_unicode(zmq.IDENTITY, 'utf16') + self.assertEqual(identu, identu2) + time.sleep(0.1) # wait for connection/subscription + p.send_unicode(topic,zmq.SNDMORE) + p.send_unicode(topic*2, encoding='latin-1') + self.assertEqual(topic, s.recv_unicode()) + self.assertEqual(topic*2, s.recv_unicode(encoding='latin-1')) + + def test_int_sockopts(self): + "test integer sockopts" + v = zmq.zmq_version_info() + if v < (3,0): + default_hwm = 0 + else: + default_hwm = 1000 + p,s = self.create_bound_pair(zmq.PUB, zmq.SUB) + p.setsockopt(zmq.LINGER, 0) + self.assertEqual(p.getsockopt(zmq.LINGER), 0) + p.setsockopt(zmq.LINGER, -1) + self.assertEqual(p.getsockopt(zmq.LINGER), -1) + self.assertEqual(p.hwm, default_hwm) + p.hwm = 11 + self.assertEqual(p.hwm, 11) + # p.setsockopt(zmq.EVENTS, zmq.POLLIN) + self.assertEqual(p.getsockopt(zmq.EVENTS), zmq.POLLOUT) + self.assertRaisesErrno(zmq.EINVAL, p.setsockopt,zmq.EVENTS, 2**7-1) + self.assertEqual(p.getsockopt(zmq.TYPE), p.socket_type) + self.assertEqual(p.getsockopt(zmq.TYPE), zmq.PUB) + self.assertEqual(s.getsockopt(zmq.TYPE), s.socket_type) + self.assertEqual(s.getsockopt(zmq.TYPE), zmq.SUB) + + # check for overflow / wrong type: + errors = [] + backref = {} + constants = zmq.constants + for name in constants.__all__: + value = getattr(constants, name) + if isinstance(value, int): + backref[value] = name + for opt in zmq.constants.int_sockopts.union(zmq.constants.int64_sockopts): + sopt = backref[opt] + if sopt.startswith(( + 'ROUTER', 'XPUB', 'TCP', 'FAIL', + 'REQ_', 'CURVE_', 'PROBE_ROUTER', + 'IPC_FILTER', 'GSSAPI', + )): + # some sockopts are write-only + continue + try: + n = p.getsockopt(opt) + except zmq.ZMQError as e: + errors.append("getsockopt(zmq.%s) raised '%s'."%(sopt, e)) + else: + if n > 2**31: + errors.append("getsockopt(zmq.%s) returned a ridiculous value." + " It is probably the wrong type."%sopt) + if errors: + self.fail('\n'.join([''] + errors)) + + def test_bad_sockopts(self): + """Test that appropriate errors are raised on bad socket options""" + s = self.context.socket(zmq.PUB) + self.sockets.append(s) + s.setsockopt(zmq.LINGER, 0) + # unrecognized int sockopts pass through to libzmq, and should raise EINVAL + self.assertRaisesErrno(zmq.EINVAL, s.setsockopt, 9999, 5) + self.assertRaisesErrno(zmq.EINVAL, s.getsockopt, 9999) + # but only int sockopts are allowed through this way, otherwise raise a TypeError + self.assertRaises(TypeError, s.setsockopt, 9999, b"5") + # some sockopts are valid in general, but not on every socket: + self.assertRaisesErrno(zmq.EINVAL, s.setsockopt, zmq.SUBSCRIBE, b'hi') + + def test_sockopt_roundtrip(self): + "test set/getsockopt roundtrip." + p = self.context.socket(zmq.PUB) + self.sockets.append(p) + p.setsockopt(zmq.LINGER, 11) + self.assertEqual(p.getsockopt(zmq.LINGER), 11) + + def test_send_unicode(self): + "test sending unicode objects" + a,b = self.create_bound_pair(zmq.PAIR, zmq.PAIR) + self.sockets.extend([a,b]) + u = "çπ§" + if str is not unicode: + u = u.decode('utf8') + self.assertRaises(TypeError, a.send, u,copy=False) + self.assertRaises(TypeError, a.send, u,copy=True) + a.send_unicode(u) + s = b.recv() + self.assertEqual(s,u.encode('utf8')) + self.assertEqual(s.decode('utf8'),u) + a.send_unicode(u,encoding='utf16') + s = b.recv_unicode(encoding='utf16') + self.assertEqual(s,u) + + @skip_pypy + def test_tracker(self): + "test the MessageTracker object for tracking when zmq is done with a buffer" + addr = 'tcp://127.0.0.1' + a = self.context.socket(zmq.PUB) + port = a.bind_to_random_port(addr) + a.close() + iface = "%s:%i"%(addr,port) + a = self.context.socket(zmq.PAIR) + # a.setsockopt(zmq.IDENTITY, b"a") + b = self.context.socket(zmq.PAIR) + self.sockets.extend([a,b]) + a.connect(iface) + time.sleep(0.1) + p1 = a.send(b'something', copy=False, track=True) + self.assertTrue(isinstance(p1, zmq.MessageTracker)) + self.assertFalse(p1.done) + p2 = a.send_multipart([b'something', b'else'], copy=False, track=True) + self.assert_(isinstance(p2, zmq.MessageTracker)) + self.assertEqual(p2.done, False) + self.assertEqual(p1.done, False) + + b.bind(iface) + msg = b.recv_multipart() + for i in range(10): + if p1.done: + break + time.sleep(0.1) + self.assertEqual(p1.done, True) + self.assertEqual(msg, [b'something']) + msg = b.recv_multipart() + for i in range(10): + if p2.done: + break + time.sleep(0.1) + self.assertEqual(p2.done, True) + self.assertEqual(msg, [b'something', b'else']) + m = zmq.Frame(b"again", track=True) + self.assertEqual(m.tracker.done, False) + p1 = a.send(m, copy=False) + p2 = a.send(m, copy=False) + self.assertEqual(m.tracker.done, False) + self.assertEqual(p1.done, False) + self.assertEqual(p2.done, False) + msg = b.recv_multipart() + self.assertEqual(m.tracker.done, False) + self.assertEqual(msg, [b'again']) + msg = b.recv_multipart() + self.assertEqual(m.tracker.done, False) + self.assertEqual(msg, [b'again']) + self.assertEqual(p1.done, False) + self.assertEqual(p2.done, False) + pm = m.tracker + del m + for i in range(10): + if p1.done: + break + time.sleep(0.1) + self.assertEqual(p1.done, True) + self.assertEqual(p2.done, True) + m = zmq.Frame(b'something', track=False) + self.assertRaises(ValueError, a.send, m, copy=False, track=True) + + + def test_close(self): + ctx = self.Context() + s = ctx.socket(zmq.PUB) + s.close() + self.assertRaisesErrno(zmq.ENOTSOCK, s.bind, b'') + self.assertRaisesErrno(zmq.ENOTSOCK, s.connect, b'') + self.assertRaisesErrno(zmq.ENOTSOCK, s.setsockopt, zmq.SUBSCRIBE, b'') + self.assertRaisesErrno(zmq.ENOTSOCK, s.send, b'asdf') + self.assertRaisesErrno(zmq.ENOTSOCK, s.recv) + del ctx + + def test_attr(self): + """set setting/getting sockopts as attributes""" + s = self.context.socket(zmq.DEALER) + self.sockets.append(s) + linger = 10 + s.linger = linger + self.assertEqual(linger, s.linger) + self.assertEqual(linger, s.getsockopt(zmq.LINGER)) + self.assertEqual(s.fd, s.getsockopt(zmq.FD)) + + def test_bad_attr(self): + s = self.context.socket(zmq.DEALER) + self.sockets.append(s) + try: + s.apple='foo' + except AttributeError: + pass + else: + self.fail("bad setattr should have raised AttributeError") + try: + s.apple + except AttributeError: + pass + else: + self.fail("bad getattr should have raised AttributeError") + + def test_subclass(self): + """subclasses can assign attributes""" + class S(zmq.Socket): + a = None + def __init__(self, *a, **kw): + self.a=-1 + super(S, self).__init__(*a, **kw) + + s = S(self.context, zmq.REP) + self.sockets.append(s) + self.assertEqual(s.a, -1) + s.a=1 + self.assertEqual(s.a, 1) + a=s.a + self.assertEqual(a, 1) + + def test_recv_multipart(self): + a,b = self.create_bound_pair() + msg = b'hi' + for i in range(3): + a.send(msg) + time.sleep(0.1) + for i in range(3): + self.assertEqual(b.recv_multipart(), [msg]) + + def test_close_after_destroy(self): + """s.close() after ctx.destroy() should be fine""" + ctx = self.Context() + s = ctx.socket(zmq.REP) + ctx.destroy() + # reaper is not instantaneous + time.sleep(1e-2) + s.close() + self.assertTrue(s.closed) + + def test_poll(self): + a,b = self.create_bound_pair() + tic = time.time() + evt = a.poll(50) + self.assertEqual(evt, 0) + evt = a.poll(50, zmq.POLLOUT) + self.assertEqual(evt, zmq.POLLOUT) + msg = b'hi' + a.send(msg) + evt = b.poll(50) + self.assertEqual(evt, zmq.POLLIN) + msg2 = self.recv(b) + evt = b.poll(50) + self.assertEqual(evt, 0) + self.assertEqual(msg2, msg) + + def test_ipc_path_max_length(self): + """IPC_PATH_MAX_LEN is a sensible value""" + if zmq.IPC_PATH_MAX_LEN == 0: + raise SkipTest("IPC_PATH_MAX_LEN undefined") + + msg = "Surprising value for IPC_PATH_MAX_LEN: %s" % zmq.IPC_PATH_MAX_LEN + self.assertTrue(zmq.IPC_PATH_MAX_LEN > 30, msg) + self.assertTrue(zmq.IPC_PATH_MAX_LEN < 1025, msg) + + def test_ipc_path_max_length_msg(self): + if zmq.IPC_PATH_MAX_LEN == 0: + raise SkipTest("IPC_PATH_MAX_LEN undefined") + + s = self.context.socket(zmq.PUB) + self.sockets.append(s) + try: + s.bind('ipc://{0}'.format('a' * (zmq.IPC_PATH_MAX_LEN + 1))) + except zmq.ZMQError as e: + self.assertTrue(str(zmq.IPC_PATH_MAX_LEN) in e.strerror) + + def test_hwm(self): + zmq3 = zmq.zmq_version_info()[0] >= 3 + for stype in (zmq.PUB, zmq.ROUTER, zmq.SUB, zmq.REQ, zmq.DEALER): + s = self.context.socket(stype) + s.hwm = 100 + self.assertEqual(s.hwm, 100) + if zmq3: + try: + self.assertEqual(s.sndhwm, 100) + except AttributeError: + pass + try: + self.assertEqual(s.rcvhwm, 100) + except AttributeError: + pass + s.close() + + def test_shadow(self): + p = self.socket(zmq.PUSH) + p.bind("tcp://127.0.0.1:5555") + p2 = zmq.Socket.shadow(p.underlying) + self.assertEqual(p.underlying, p2.underlying) + s = self.socket(zmq.PULL) + s2 = zmq.Socket.shadow(s.underlying) + self.assertNotEqual(s.underlying, p.underlying) + self.assertEqual(s.underlying, s2.underlying) + s2.connect("tcp://127.0.0.1:5555") + sent = b'hi' + p2.send(sent) + rcvd = self.recv(s2) + self.assertEqual(rcvd, sent) + + def test_shadow_pyczmq(self): + try: + from pyczmq import zctx, zsocket + except Exception: + raise SkipTest("Requires pyczmq") + + ctx = zctx.new() + ca = zsocket.new(ctx, zmq.PUSH) + cb = zsocket.new(ctx, zmq.PULL) + a = zmq.Socket.shadow(ca) + b = zmq.Socket.shadow(cb) + a.bind("inproc://a") + b.connect("inproc://a") + a.send(b'hi') + rcvd = self.recv(b) + self.assertEqual(rcvd, b'hi') + + +if have_gevent: + import gevent + + class TestSocketGreen(GreenTest, TestSocket): + test_bad_attr = GreenTest.skip_green + test_close_after_destroy = GreenTest.skip_green + + def test_timeout(self): + a,b = self.create_bound_pair() + g = gevent.spawn_later(0.5, lambda: a.send(b'hi')) + timeout = gevent.Timeout(0.1) + timeout.start() + self.assertRaises(gevent.Timeout, b.recv) + g.kill() + + @skip_if(not hasattr(zmq, 'RCVTIMEO')) + def test_warn_set_timeo(self): + s = self.context.socket(zmq.REQ) + with warnings.catch_warnings(record=True) as w: + s.rcvtimeo = 5 + s.close() + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, UserWarning) + + + @skip_if(not hasattr(zmq, 'SNDTIMEO')) + def test_warn_get_timeo(self): + s = self.context.socket(zmq.REQ) + with warnings.catch_warnings(record=True) as w: + s.sndtimeo + s.close() + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, UserWarning) diff --git a/scripts/external_libs/zmq/tests/test_stopwatch.py b/scripts/external_libs/zmq/tests/test_stopwatch.py new file mode 100644 index 00000000..49fb79f2 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_stopwatch.py @@ -0,0 +1,42 @@ +# -*- coding: utf8 -*- +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import sys +import time + +from unittest import TestCase + +from zmq import Stopwatch, ZMQError + +if sys.version_info[0] >= 3: + long = int + +class TestStopWatch(TestCase): + + def test_stop_long(self): + """Ensure stop returns a long int.""" + watch = Stopwatch() + watch.start() + us = watch.stop() + self.assertTrue(isinstance(us, long)) + + def test_stop_microseconds(self): + """Test that stop/sleep have right units.""" + watch = Stopwatch() + watch.start() + tic = time.time() + watch.sleep(1) + us = watch.stop() + toc = time.time() + self.assertAlmostEqual(us/1e6,(toc-tic),places=0) + + def test_double_stop(self): + """Test error raised on multiple calls to stop.""" + watch = Stopwatch() + watch.start() + watch.stop() + self.assertRaises(ZMQError, watch.stop) + self.assertRaises(ZMQError, watch.stop) + diff --git a/scripts/external_libs/zmq/tests/test_version.py b/scripts/external_libs/zmq/tests/test_version.py new file mode 100644 index 00000000..6ebebf30 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_version.py @@ -0,0 +1,44 @@ +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +from unittest import TestCase +import zmq +from zmq.sugar import version + + +class TestVersion(TestCase): + + def test_pyzmq_version(self): + vs = zmq.pyzmq_version() + vs2 = zmq.__version__ + self.assertTrue(isinstance(vs, str)) + if zmq.__revision__: + self.assertEqual(vs, '@'.join(vs2, zmq.__revision__)) + else: + self.assertEqual(vs, vs2) + if version.VERSION_EXTRA: + self.assertTrue(version.VERSION_EXTRA in vs) + self.assertTrue(version.VERSION_EXTRA in vs2) + + def test_pyzmq_version_info(self): + info = zmq.pyzmq_version_info() + self.assertTrue(isinstance(info, tuple)) + for n in info[:3]: + self.assertTrue(isinstance(n, int)) + if version.VERSION_EXTRA: + self.assertEqual(len(info), 4) + self.assertEqual(info[-1], float('inf')) + else: + self.assertEqual(len(info), 3) + + def test_zmq_version_info(self): + info = zmq.zmq_version_info() + self.assertTrue(isinstance(info, tuple)) + for n in info[:3]: + self.assertTrue(isinstance(n, int)) + + def test_zmq_version(self): + v = zmq.zmq_version() + self.assertTrue(isinstance(v, str)) + diff --git a/scripts/external_libs/zmq/tests/test_win32_shim.py b/scripts/external_libs/zmq/tests/test_win32_shim.py new file mode 100644 index 00000000..55657bda --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_win32_shim.py @@ -0,0 +1,56 @@ +from __future__ import print_function + +import os + +from functools import wraps +from zmq.tests import BaseZMQTestCase +from zmq.utils.win32 import allow_interrupt + + +def count_calls(f): + @wraps(f) + def _(*args, **kwds): + try: + return f(*args, **kwds) + finally: + _.__calls__ += 1 + _.__calls__ = 0 + return _ + + +class TestWindowsConsoleControlHandler(BaseZMQTestCase): + + def test_handler(self): + @count_calls + def interrupt_polling(): + print('Caught CTRL-C!') + + if os.name == 'nt': + from ctypes import windll + from ctypes.wintypes import BOOL, DWORD + + kernel32 = windll.LoadLibrary('kernel32') + + # <http://msdn.microsoft.com/en-us/library/ms683155.aspx> + GenerateConsoleCtrlEvent = kernel32.GenerateConsoleCtrlEvent + GenerateConsoleCtrlEvent.argtypes = (DWORD, DWORD) + GenerateConsoleCtrlEvent.restype = BOOL + + try: + # Simulate CTRL-C event while handler is active. + with allow_interrupt(interrupt_polling): + result = GenerateConsoleCtrlEvent(0, 0) + if result == 0: + raise WindowsError + except KeyboardInterrupt: + pass + else: + self.fail('Expecting `KeyboardInterrupt` exception!') + + # Make sure our handler was called. + self.assertEqual(interrupt_polling.__calls__, 1) + else: + # On non-Windows systems, this utility is just a no-op! + with allow_interrupt(interrupt_polling): + pass + self.assertEqual(interrupt_polling.__calls__, 0) diff --git a/scripts/external_libs/zmq/tests/test_z85.py b/scripts/external_libs/zmq/tests/test_z85.py new file mode 100644 index 00000000..8a73cb4d --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_z85.py @@ -0,0 +1,63 @@ +# -*- coding: utf8 -*- +"""Test Z85 encoding + +confirm values and roundtrip with test values from the reference implementation. +""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from unittest import TestCase +from zmq.utils import z85 + + +class TestZ85(TestCase): + + def test_client_public(self): + client_public = \ + b"\xBB\x88\x47\x1D\x65\xE2\x65\x9B" \ + b"\x30\xC5\x5A\x53\x21\xCE\xBB\x5A" \ + b"\xAB\x2B\x70\xA3\x98\x64\x5C\x26" \ + b"\xDC\xA2\xB2\xFC\xB4\x3F\xC5\x18" + encoded = z85.encode(client_public) + + self.assertEqual(encoded, b"Yne@$w-vo<fVvi]a<NY6T1ed:M$fCG*[IaLV{hID") + decoded = z85.decode(encoded) + self.assertEqual(decoded, client_public) + + def test_client_secret(self): + client_secret = \ + b"\x7B\xB8\x64\xB4\x89\xAF\xA3\x67" \ + b"\x1F\xBE\x69\x10\x1F\x94\xB3\x89" \ + b"\x72\xF2\x48\x16\xDF\xB0\x1B\x51" \ + b"\x65\x6B\x3F\xEC\x8D\xFD\x08\x88" + encoded = z85.encode(client_secret) + + self.assertEqual(encoded, b"D:)Q[IlAW!ahhC2ac:9*A}h:p?([4%wOTJ%JR%cs") + decoded = z85.decode(encoded) + self.assertEqual(decoded, client_secret) + + def test_server_public(self): + server_public = \ + b"\x54\xFC\xBA\x24\xE9\x32\x49\x96" \ + b"\x93\x16\xFB\x61\x7C\x87\x2B\xB0" \ + b"\xC1\xD1\xFF\x14\x80\x04\x27\xC5" \ + b"\x94\xCB\xFA\xCF\x1B\xC2\xD6\x52" + encoded = z85.encode(server_public) + + self.assertEqual(encoded, b"rq:rM>}U?@Lns47E1%kR.o@n%FcmmsL/@{H8]yf7") + decoded = z85.decode(encoded) + self.assertEqual(decoded, server_public) + + def test_server_secret(self): + server_secret = \ + b"\x8E\x0B\xDD\x69\x76\x28\xB9\x1D" \ + b"\x8F\x24\x55\x87\xEE\x95\xC5\xB0" \ + b"\x4D\x48\x96\x3F\x79\x25\x98\x77" \ + b"\xB4\x9C\xD9\x06\x3A\xEA\xD3\xB7" + encoded = z85.encode(server_secret) + + self.assertEqual(encoded, b"JTKVSB%%)wK0E.X)V>+}o?pNmC{O&4W4b!Ni{Lh6") + decoded = z85.decode(encoded) + self.assertEqual(decoded, server_secret) + diff --git a/scripts/external_libs/zmq/tests/test_zmqstream.py b/scripts/external_libs/zmq/tests/test_zmqstream.py new file mode 100644 index 00000000..cdb3a171 --- /dev/null +++ b/scripts/external_libs/zmq/tests/test_zmqstream.py @@ -0,0 +1,34 @@ +# -*- coding: utf8 -*- +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import sys +import time + +from unittest import TestCase + +import zmq +from zmq.eventloop import ioloop, zmqstream + +class TestZMQStream(TestCase): + + def setUp(self): + self.context = zmq.Context() + self.socket = self.context.socket(zmq.REP) + self.loop = ioloop.IOLoop.instance() + self.stream = zmqstream.ZMQStream(self.socket) + + def tearDown(self): + self.socket.close() + self.context.term() + + def test_callable_check(self): + """Ensure callable check works (py3k).""" + + self.stream.on_send(lambda *args: None) + self.stream.on_recv(lambda *args: None) + self.assertRaises(AssertionError, self.stream.on_recv, 1) + self.assertRaises(AssertionError, self.stream.on_send, 1) + self.assertRaises(AssertionError, self.stream.on_recv, zmq) + diff --git a/scripts/external_libs/zmq/utils/__init__.py b/scripts/external_libs/zmq/utils/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/scripts/external_libs/zmq/utils/__init__.py diff --git a/scripts/external_libs/zmq/utils/buffers.pxd b/scripts/external_libs/zmq/utils/buffers.pxd new file mode 100644 index 00000000..998aa551 --- /dev/null +++ b/scripts/external_libs/zmq/utils/buffers.pxd @@ -0,0 +1,313 @@ +"""Python version-independent methods for C/Python buffers. + +This file was copied and adapted from mpi4py. + +Authors +------- +* MinRK +""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Lisandro Dalcin +# All rights reserved. +# Used under BSD License: http://www.opensource.org/licenses/bsd-license.php +# +# Retrieval: +# Jul 23, 2010 18:00 PST (r539) +# http://code.google.com/p/mpi4py/source/browse/trunk/src/MPI/asbuffer.pxi +# +# Modifications from original: +# Copyright (c) 2010-2012 Brian Granger, Min Ragan-Kelley +# +# Distributed under the terms of the New BSD License. The full license is in +# the file COPYING.BSD, distributed as part of this software. +#----------------------------------------------------------------------------- + + +#----------------------------------------------------------------------------- +# Python includes. +#----------------------------------------------------------------------------- + +# get version-independent aliases: +cdef extern from "pyversion_compat.h": + pass + +# Python 3 buffer interface (PEP 3118) +cdef extern from "Python.h": + int PY_MAJOR_VERSION + int PY_MINOR_VERSION + ctypedef int Py_ssize_t + ctypedef struct PyMemoryViewObject: + pass + ctypedef struct Py_buffer: + void *buf + Py_ssize_t len + int readonly + char *format + int ndim + Py_ssize_t *shape + Py_ssize_t *strides + Py_ssize_t *suboffsets + Py_ssize_t itemsize + void *internal + cdef enum: + PyBUF_SIMPLE + PyBUF_WRITABLE + PyBUF_FORMAT + PyBUF_ANY_CONTIGUOUS + int PyObject_CheckBuffer(object) + int PyObject_GetBuffer(object, Py_buffer *, int) except -1 + void PyBuffer_Release(Py_buffer *) + + int PyBuffer_FillInfo(Py_buffer *view, object obj, void *buf, + Py_ssize_t len, int readonly, int infoflags) except -1 + object PyMemoryView_FromBuffer(Py_buffer *info) + + object PyMemoryView_FromObject(object) + +# Python 2 buffer interface (legacy) +cdef extern from "Python.h": + ctypedef void const_void "const void" + Py_ssize_t Py_END_OF_BUFFER + int PyObject_CheckReadBuffer(object) + int PyObject_AsReadBuffer (object, const_void **, Py_ssize_t *) except -1 + int PyObject_AsWriteBuffer(object, void **, Py_ssize_t *) except -1 + + object PyBuffer_FromMemory(void *ptr, Py_ssize_t s) + object PyBuffer_FromReadWriteMemory(void *ptr, Py_ssize_t s) + + object PyBuffer_FromObject(object, Py_ssize_t offset, Py_ssize_t size) + object PyBuffer_FromReadWriteObject(object, Py_ssize_t offset, Py_ssize_t size) + + +#----------------------------------------------------------------------------- +# asbuffer: C buffer from python object +#----------------------------------------------------------------------------- + + +cdef inline int memoryview_available(): + return PY_MAJOR_VERSION >= 3 or (PY_MAJOR_VERSION >=2 and PY_MINOR_VERSION >= 7) + +cdef inline int oldstyle_available(): + return PY_MAJOR_VERSION < 3 + + +cdef inline int check_buffer(object ob): + """Version independent check for whether an object is a buffer. + + Parameters + ---------- + object : object + Any Python object + + Returns + ------- + int : 0 if no buffer interface, 3 if newstyle buffer interface, 2 if oldstyle. + """ + if PyObject_CheckBuffer(ob): + return 3 + if oldstyle_available(): + return PyObject_CheckReadBuffer(ob) and 2 + return 0 + + +cdef inline object asbuffer(object ob, int writable, int format, + void **base, Py_ssize_t *size, + Py_ssize_t *itemsize): + """Turn an object into a C buffer in a Python version-independent way. + + Parameters + ---------- + ob : object + The object to be turned into a buffer. + Must provide a Python Buffer interface + writable : int + Whether the resulting buffer should be allowed to write + to the object. + format : int + The format of the buffer. See Python buffer docs. + base : void ** + The pointer that will be used to store the resulting C buffer. + size : Py_ssize_t * + The size of the buffer(s). + itemsize : Py_ssize_t * + The size of an item, if the buffer is non-contiguous. + + Returns + ------- + An object describing the buffer format. Generally a str, such as 'B'. + """ + + cdef void *bptr = NULL + cdef Py_ssize_t blen = 0, bitemlen = 0 + cdef Py_buffer view + cdef int flags = PyBUF_SIMPLE + cdef int mode = 0 + + bfmt = None + + mode = check_buffer(ob) + if mode == 0: + raise TypeError("%r does not provide a buffer interface."%ob) + + if mode == 3: + flags = PyBUF_ANY_CONTIGUOUS + if writable: + flags |= PyBUF_WRITABLE + if format: + flags |= PyBUF_FORMAT + PyObject_GetBuffer(ob, &view, flags) + bptr = view.buf + blen = view.len + if format: + if view.format != NULL: + bfmt = view.format + bitemlen = view.itemsize + PyBuffer_Release(&view) + else: # oldstyle + if writable: + PyObject_AsWriteBuffer(ob, &bptr, &blen) + else: + PyObject_AsReadBuffer(ob, <const_void **>&bptr, &blen) + if format: + try: # numpy.ndarray + dtype = ob.dtype + bfmt = dtype.char + bitemlen = dtype.itemsize + except AttributeError: + try: # array.array + bfmt = ob.typecode + bitemlen = ob.itemsize + except AttributeError: + if isinstance(ob, bytes): + bfmt = b"B" + bitemlen = 1 + else: + # nothing found + bfmt = None + bitemlen = 0 + if base: base[0] = <void *>bptr + if size: size[0] = <Py_ssize_t>blen + if itemsize: itemsize[0] = <Py_ssize_t>bitemlen + + if PY_MAJOR_VERSION >= 3 and bfmt is not None: + return bfmt.decode('ascii') + return bfmt + + +cdef inline object asbuffer_r(object ob, void **base, Py_ssize_t *size): + """Wrapper for standard calls to asbuffer with a readonly buffer.""" + asbuffer(ob, 0, 0, base, size, NULL) + return ob + + +cdef inline object asbuffer_w(object ob, void **base, Py_ssize_t *size): + """Wrapper for standard calls to asbuffer with a writable buffer.""" + asbuffer(ob, 1, 0, base, size, NULL) + return ob + +#------------------------------------------------------------------------------ +# frombuffer: python buffer/view from C buffer +#------------------------------------------------------------------------------ + + +cdef inline object frombuffer_3(void *ptr, Py_ssize_t s, int readonly): + """Python 3 version of frombuffer. + + This is the Python 3 model, but will work on Python >= 2.6. Currently, + we use it only on >= 3.0. + """ + cdef Py_buffer pybuf + cdef Py_ssize_t *shape = [s] + cdef str astr="" + PyBuffer_FillInfo(&pybuf, astr, ptr, s, readonly, PyBUF_SIMPLE) + pybuf.format = "B" + pybuf.shape = shape + return PyMemoryView_FromBuffer(&pybuf) + + +cdef inline object frombuffer_2(void *ptr, Py_ssize_t s, int readonly): + """Python 2 version of frombuffer. + + This must be used for Python <= 2.6, but we use it for all Python < 3. + """ + + if oldstyle_available(): + if readonly: + return PyBuffer_FromMemory(ptr, s) + else: + return PyBuffer_FromReadWriteMemory(ptr, s) + else: + raise NotImplementedError("Old style buffers not available.") + + +cdef inline object frombuffer(void *ptr, Py_ssize_t s, int readonly): + """Create a Python Buffer/View of a C array. + + Parameters + ---------- + ptr : void * + Pointer to the array to be copied. + s : size_t + Length of the buffer. + readonly : int + whether the resulting object should be allowed to write to the buffer. + + Returns + ------- + Python Buffer/View of the C buffer. + """ + # oldstyle first priority for now + if oldstyle_available(): + return frombuffer_2(ptr, s, readonly) + else: + return frombuffer_3(ptr, s, readonly) + + +cdef inline object frombuffer_r(void *ptr, Py_ssize_t s): + """Wrapper for readonly view frombuffer.""" + return frombuffer(ptr, s, 1) + + +cdef inline object frombuffer_w(void *ptr, Py_ssize_t s): + """Wrapper for writable view frombuffer.""" + return frombuffer(ptr, s, 0) + +#------------------------------------------------------------------------------ +# viewfromobject: python buffer/view from python object, refcounts intact +# frombuffer(asbuffer(obj)) would lose track of refs +#------------------------------------------------------------------------------ + +cdef inline object viewfromobject(object obj, int readonly): + """Construct a Python Buffer/View object from another Python object. + + This work in a Python version independent manner. + + Parameters + ---------- + obj : object + The input object to be cast as a buffer + readonly : int + Whether the result should be prevented from overwriting the original. + + Returns + ------- + Buffer/View of the original object. + """ + if not memoryview_available(): + if readonly: + return PyBuffer_FromObject(obj, 0, Py_END_OF_BUFFER) + else: + return PyBuffer_FromReadWriteObject(obj, 0, Py_END_OF_BUFFER) + else: + return PyMemoryView_FromObject(obj) + + +cdef inline object viewfromobject_r(object obj): + """Wrapper for readonly viewfromobject.""" + return viewfromobject(obj, 1) + + +cdef inline object viewfromobject_w(object obj): + """Wrapper for writable viewfromobject.""" + return viewfromobject(obj, 0) diff --git a/scripts/external_libs/zmq/utils/compiler.json b/scripts/external_libs/zmq/utils/compiler.json new file mode 100644 index 00000000..e58fc130 --- /dev/null +++ b/scripts/external_libs/zmq/utils/compiler.json @@ -0,0 +1,24 @@ +{ + "extra_link_args": [], + "define_macros": [ + [ + "HAVE_SYS_UN_H", + 1 + ] + ], + "runtime_library_dirs": [ + "$ORIGIN/.." + ], + "libraries": [ + "zmq" + ], + "library_dirs": [ + "zmq" + ], + "include_dirs": [ + "/auto/srg-sce-swinfra-usr/emb/users/hhaim/work/depot/asr1k/emb/private/bpsim/main/src/zmq/include", + "zmq/utils", + "zmq/backend/cython", + "zmq/devices" + ] +}
\ No newline at end of file diff --git a/scripts/external_libs/zmq/utils/config.json b/scripts/external_libs/zmq/utils/config.json new file mode 100644 index 00000000..1e4611f9 --- /dev/null +++ b/scripts/external_libs/zmq/utils/config.json @@ -0,0 +1,10 @@ +{ + "have_sys_un_h": true, + "zmq_prefix": "/auto/srg-sce-swinfra-usr/emb/users/hhaim/work/depot/asr1k/emb/private/bpsim/main/src/zmq", + "no_libzmq_extension": true, + "libzmq_extension": false, + "easy_install": {}, + "bdist_egg": {}, + "skip_check_zmq": false, + "build_ext": {} +}
\ No newline at end of file diff --git a/scripts/external_libs/zmq/utils/constant_names.py b/scripts/external_libs/zmq/utils/constant_names.py new file mode 100644 index 00000000..47da9dc2 --- /dev/null +++ b/scripts/external_libs/zmq/utils/constant_names.py @@ -0,0 +1,365 @@ +"""0MQ Constant names""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +# dictionaries of constants new or removed in particular versions + +new_in = { + (2,2,0) : [ + 'RCVTIMEO', + 'SNDTIMEO', + ], + (3,2,2) : [ + # errnos + 'EMSGSIZE', + 'EAFNOSUPPORT', + 'ENETUNREACH', + 'ECONNABORTED', + 'ECONNRESET', + 'ENOTCONN', + 'ETIMEDOUT', + 'EHOSTUNREACH', + 'ENETRESET', + + # ctx opts + 'IO_THREADS', + 'MAX_SOCKETS', + 'IO_THREADS_DFLT', + 'MAX_SOCKETS_DFLT', + + # socket opts + 'ROUTER_BEHAVIOR', + 'ROUTER_MANDATORY', + 'FAIL_UNROUTABLE', + 'TCP_KEEPALIVE', + 'TCP_KEEPALIVE_CNT', + 'TCP_KEEPALIVE_IDLE', + 'TCP_KEEPALIVE_INTVL', + 'DELAY_ATTACH_ON_CONNECT', + 'XPUB_VERBOSE', + + # msg opts + 'MORE', + + 'EVENT_CONNECTED', + 'EVENT_CONNECT_DELAYED', + 'EVENT_CONNECT_RETRIED', + 'EVENT_LISTENING', + 'EVENT_BIND_FAILED', + 'EVENT_ACCEPTED', + 'EVENT_ACCEPT_FAILED', + 'EVENT_CLOSED', + 'EVENT_CLOSE_FAILED', + 'EVENT_DISCONNECTED', + 'EVENT_ALL', + ], + (4,0,0) : [ + # socket types + 'STREAM', + + # socket opts + 'IMMEDIATE', + 'ROUTER_RAW', + 'IPV6', + 'MECHANISM', + 'PLAIN_SERVER', + 'PLAIN_USERNAME', + 'PLAIN_PASSWORD', + 'CURVE_SERVER', + 'CURVE_PUBLICKEY', + 'CURVE_SECRETKEY', + 'CURVE_SERVERKEY', + 'PROBE_ROUTER', + 'REQ_RELAXED', + 'REQ_CORRELATE', + 'CONFLATE', + 'ZAP_DOMAIN', + + # security + 'NULL', + 'PLAIN', + 'CURVE', + + # events + 'EVENT_MONITOR_STOPPED', + ], + (4,1,0) : [ + # ctx opts + 'SOCKET_LIMIT', + 'THREAD_PRIORITY', + 'THREAD_PRIORITY_DFLT', + 'THREAD_SCHED_POLICY', + 'THREAD_SCHED_POLICY_DFLT', + + # socket opts + 'ROUTER_HANDOVER', + 'TOS', + 'IPC_FILTER_PID', + 'IPC_FILTER_UID', + 'IPC_FILTER_GID', + 'CONNECT_RID', + 'GSSAPI_SERVER', + 'GSSAPI_PRINCIPAL', + 'GSSAPI_SERVICE_PRINCIPAL', + 'GSSAPI_PLAINTEXT', + 'HANDSHAKE_IVL', + 'IDENTITY_FD', + 'XPUB_NODROP', + 'SOCKS_PROXY', + + # msg opts + 'SRCFD', + 'SHARED', + + # security + 'GSSAPI', + + ], +} + + +removed_in = { + (3,2,2) : [ + 'UPSTREAM', + 'DOWNSTREAM', + + 'HWM', + 'SWAP', + 'MCAST_LOOP', + 'RECOVERY_IVL_MSEC', + ] +} + +# collections of zmq constant names based on their role +# base names have no specific use +# opt names are validated in get/set methods of various objects + +base_names = [ + # base + 'VERSION', + 'VERSION_MAJOR', + 'VERSION_MINOR', + 'VERSION_PATCH', + 'NOBLOCK', + 'DONTWAIT', + + 'POLLIN', + 'POLLOUT', + 'POLLERR', + + 'SNDMORE', + + 'STREAMER', + 'FORWARDER', + 'QUEUE', + + 'IO_THREADS_DFLT', + 'MAX_SOCKETS_DFLT', + 'POLLITEMS_DFLT', + 'THREAD_PRIORITY_DFLT', + 'THREAD_SCHED_POLICY_DFLT', + + # socktypes + 'PAIR', + 'PUB', + 'SUB', + 'REQ', + 'REP', + 'DEALER', + 'ROUTER', + 'XREQ', + 'XREP', + 'PULL', + 'PUSH', + 'XPUB', + 'XSUB', + 'UPSTREAM', + 'DOWNSTREAM', + 'STREAM', + + # events + 'EVENT_CONNECTED', + 'EVENT_CONNECT_DELAYED', + 'EVENT_CONNECT_RETRIED', + 'EVENT_LISTENING', + 'EVENT_BIND_FAILED', + 'EVENT_ACCEPTED', + 'EVENT_ACCEPT_FAILED', + 'EVENT_CLOSED', + 'EVENT_CLOSE_FAILED', + 'EVENT_DISCONNECTED', + 'EVENT_ALL', + 'EVENT_MONITOR_STOPPED', + + # security + 'NULL', + 'PLAIN', + 'CURVE', + 'GSSAPI', + + ## ERRNO + # Often used (these are alse in errno.) + 'EAGAIN', + 'EINVAL', + 'EFAULT', + 'ENOMEM', + 'ENODEV', + 'EMSGSIZE', + 'EAFNOSUPPORT', + 'ENETUNREACH', + 'ECONNABORTED', + 'ECONNRESET', + 'ENOTCONN', + 'ETIMEDOUT', + 'EHOSTUNREACH', + 'ENETRESET', + + # For Windows compatability + 'HAUSNUMERO', + 'ENOTSUP', + 'EPROTONOSUPPORT', + 'ENOBUFS', + 'ENETDOWN', + 'EADDRINUSE', + 'EADDRNOTAVAIL', + 'ECONNREFUSED', + 'EINPROGRESS', + 'ENOTSOCK', + + # 0MQ Native + 'EFSM', + 'ENOCOMPATPROTO', + 'ETERM', + 'EMTHREAD', +] + +int64_sockopt_names = [ + 'AFFINITY', + 'MAXMSGSIZE', + + # sockopts removed in 3.0.0 + 'HWM', + 'SWAP', + 'MCAST_LOOP', + 'RECOVERY_IVL_MSEC', +] + +bytes_sockopt_names = [ + 'IDENTITY', + 'SUBSCRIBE', + 'UNSUBSCRIBE', + 'LAST_ENDPOINT', + 'TCP_ACCEPT_FILTER', + + 'PLAIN_USERNAME', + 'PLAIN_PASSWORD', + + 'CURVE_PUBLICKEY', + 'CURVE_SECRETKEY', + 'CURVE_SERVERKEY', + 'ZAP_DOMAIN', + 'CONNECT_RID', + 'GSSAPI_PRINCIPAL', + 'GSSAPI_SERVICE_PRINCIPAL', + 'SOCKS_PROXY', +] + +fd_sockopt_names = [ + 'FD', + 'IDENTITY_FD', +] + +int_sockopt_names = [ + # sockopts + 'RECONNECT_IVL_MAX', + + # sockopts new in 2.2.0 + 'SNDTIMEO', + 'RCVTIMEO', + + # new in 3.x + 'SNDHWM', + 'RCVHWM', + 'MULTICAST_HOPS', + 'IPV4ONLY', + + 'ROUTER_BEHAVIOR', + 'TCP_KEEPALIVE', + 'TCP_KEEPALIVE_CNT', + 'TCP_KEEPALIVE_IDLE', + 'TCP_KEEPALIVE_INTVL', + 'DELAY_ATTACH_ON_CONNECT', + 'XPUB_VERBOSE', + + 'EVENTS', + 'TYPE', + 'LINGER', + 'RECONNECT_IVL', + 'BACKLOG', + + 'ROUTER_MANDATORY', + 'FAIL_UNROUTABLE', + + 'ROUTER_RAW', + 'IMMEDIATE', + 'IPV6', + 'MECHANISM', + 'PLAIN_SERVER', + 'CURVE_SERVER', + 'PROBE_ROUTER', + 'REQ_RELAXED', + 'REQ_CORRELATE', + 'CONFLATE', + 'ROUTER_HANDOVER', + 'TOS', + 'IPC_FILTER_PID', + 'IPC_FILTER_UID', + 'IPC_FILTER_GID', + 'GSSAPI_SERVER', + 'GSSAPI_PLAINTEXT', + 'HANDSHAKE_IVL', + 'XPUB_NODROP', +] + +switched_sockopt_names = [ + 'RATE', + 'RECOVERY_IVL', + 'SNDBUF', + 'RCVBUF', + 'RCVMORE', +] + +ctx_opt_names = [ + 'IO_THREADS', + 'MAX_SOCKETS', + 'SOCKET_LIMIT', + 'THREAD_PRIORITY', + 'THREAD_SCHED_POLICY', +] + +msg_opt_names = [ + 'MORE', + 'SRCFD', + 'SHARED', +] + +from itertools import chain + +all_names = list(chain( + base_names, + ctx_opt_names, + bytes_sockopt_names, + fd_sockopt_names, + int_sockopt_names, + int64_sockopt_names, + switched_sockopt_names, + msg_opt_names, +)) + +del chain + +def no_prefix(name): + """does the given constant have a ZMQ_ prefix?""" + return name.startswith('E') and not name.startswith('EVENT') + diff --git a/scripts/external_libs/zmq/utils/garbage.py b/scripts/external_libs/zmq/utils/garbage.py new file mode 100644 index 00000000..80a8725a --- /dev/null +++ b/scripts/external_libs/zmq/utils/garbage.py @@ -0,0 +1,180 @@ +"""Garbage collection thread for representing zmq refcount of Python objects +used in zero-copy sends. +""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +import atexit +import struct + +from os import getpid +from collections import namedtuple +from threading import Thread, Event, Lock +import warnings + +import zmq + + +gcref = namedtuple('gcref', ['obj', 'event']) + +class GarbageCollectorThread(Thread): + """Thread in which garbage collection actually happens.""" + def __init__(self, gc): + super(GarbageCollectorThread, self).__init__() + self.gc = gc + self.daemon = True + self.pid = getpid() + self.ready = Event() + + def run(self): + # detect fork at begining of the thread + if getpid is None or getpid() != self.pid: + self.ready.set() + return + try: + s = self.gc.context.socket(zmq.PULL) + s.linger = 0 + s.bind(self.gc.url) + finally: + self.ready.set() + + while True: + # detect fork + if getpid is None or getpid() != self.pid: + return + msg = s.recv() + if msg == b'DIE': + break + fmt = 'L' if len(msg) == 4 else 'Q' + key = struct.unpack(fmt, msg)[0] + tup = self.gc.refs.pop(key, None) + if tup and tup.event: + tup.event.set() + del tup + s.close() + + +class GarbageCollector(object): + """PyZMQ Garbage Collector + + Used for representing the reference held by libzmq during zero-copy sends. + This object holds a dictionary, keyed by Python id, + of the Python objects whose memory are currently in use by zeromq. + + When zeromq is done with the memory, it sends a message on an inproc PUSH socket + containing the packed size_t (32 or 64-bit unsigned int), + which is the key in the dict. + When the PULL socket in the gc thread receives that message, + the reference is popped from the dict, + and any tracker events that should be signaled fire. + """ + + refs = None + _context = None + _lock = None + url = "inproc://pyzmq.gc.01" + + def __init__(self, context=None): + super(GarbageCollector, self).__init__() + self.refs = {} + self.pid = None + self.thread = None + self._context = context + self._lock = Lock() + self._stay_down = False + atexit.register(self._atexit) + + @property + def context(self): + if self._context is None: + self._context = zmq.Context() + return self._context + + @context.setter + def context(self, ctx): + if self.is_alive(): + if self.refs: + warnings.warn("Replacing gc context while gc is running", RuntimeWarning) + self.stop() + self._context = ctx + + def _atexit(self): + """atexit callback + + sets _stay_down flag so that gc doesn't try to start up again in other atexit handlers + """ + self._stay_down = True + self.stop() + + def stop(self): + """stop the garbage-collection thread""" + if not self.is_alive(): + return + self._stop() + + def _stop(self): + push = self.context.socket(zmq.PUSH) + push.connect(self.url) + push.send(b'DIE') + push.close() + self.thread.join() + self.context.term() + self.refs.clear() + self.context = None + + def start(self): + """Start a new garbage collection thread. + + Creates a new zmq Context used for garbage collection. + Under most circumstances, this will only be called once per process. + """ + if self.thread is not None and self.pid != getpid(): + # It's re-starting, must free earlier thread's context + # since a fork probably broke it + self._stop() + self.pid = getpid() + self.refs = {} + self.thread = GarbageCollectorThread(self) + self.thread.start() + self.thread.ready.wait() + + def is_alive(self): + """Is the garbage collection thread currently running? + + Includes checks for process shutdown or fork. + """ + if (getpid is None or + getpid() != self.pid or + self.thread is None or + not self.thread.is_alive() + ): + return False + return True + + def store(self, obj, event=None): + """store an object and (optionally) event for zero-copy""" + if not self.is_alive(): + if self._stay_down: + return 0 + # safely start the gc thread + # use lock and double check, + # so we don't start multiple threads + with self._lock: + if not self.is_alive(): + self.start() + tup = gcref(obj, event) + theid = id(tup) + self.refs[theid] = tup + return theid + + def __del__(self): + if not self.is_alive(): + return + try: + self.stop() + except Exception as e: + raise (e) + +gc = GarbageCollector() diff --git a/scripts/external_libs/zmq/utils/getpid_compat.h b/scripts/external_libs/zmq/utils/getpid_compat.h new file mode 100644 index 00000000..47ce90fa --- /dev/null +++ b/scripts/external_libs/zmq/utils/getpid_compat.h @@ -0,0 +1,6 @@ +#ifdef _WIN32 + #include <process.h> + #define getpid _getpid +#else + #include <unistd.h> +#endif diff --git a/scripts/external_libs/zmq/utils/interop.py b/scripts/external_libs/zmq/utils/interop.py new file mode 100644 index 00000000..26c01969 --- /dev/null +++ b/scripts/external_libs/zmq/utils/interop.py @@ -0,0 +1,33 @@ +"""Utils for interoperability with other libraries. + +Just CFFI pointer casting for now. +""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + + +try: + long +except NameError: + long = int # Python 3 + + +def cast_int_addr(n): + """Cast an address to a Python int + + This could be a Python integer or a CFFI pointer + """ + if isinstance(n, (int, long)): + return n + try: + import cffi + except ImportError: + pass + else: + # from pyzmq, this is an FFI void * + ffi = cffi.FFI() + if isinstance(n, ffi.CData): + return int(ffi.cast("size_t", n)) + + raise ValueError("Cannot cast %r to int" % n) diff --git a/scripts/external_libs/zmq/utils/ipcmaxlen.h b/scripts/external_libs/zmq/utils/ipcmaxlen.h new file mode 100644 index 00000000..7218db78 --- /dev/null +++ b/scripts/external_libs/zmq/utils/ipcmaxlen.h @@ -0,0 +1,21 @@ +/* + +Platform-independant detection of IPC path max length + +Copyright (c) 2012 Godefroid Chapelle + +Distributed under the terms of the New BSD License. The full license is in +the file COPYING.BSD, distributed as part of this software. + */ + +#if defined(HAVE_SYS_UN_H) +#include "sys/un.h" +int get_ipc_path_max_len(void) { + struct sockaddr_un *dummy; + return sizeof(dummy->sun_path) - 1; +} +#else +int get_ipc_path_max_len(void) { + return 0; +} +#endif diff --git a/scripts/external_libs/zmq/utils/jsonapi.py b/scripts/external_libs/zmq/utils/jsonapi.py new file mode 100644 index 00000000..865ca6d5 --- /dev/null +++ b/scripts/external_libs/zmq/utils/jsonapi.py @@ -0,0 +1,59 @@ +"""Priority based json library imports. + +Always serializes to bytes instead of unicode for zeromq compatibility +on Python 2 and 3. + +Use ``jsonapi.loads()`` and ``jsonapi.dumps()`` for guaranteed symmetry. + +Priority: ``simplejson`` > ``jsonlib2`` > stdlib ``json`` + +``jsonapi.loads/dumps`` provide kwarg-compatibility with stdlib json. + +``jsonapi.jsonmod`` will be the module of the actual underlying implementation. +""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +from zmq.utils.strtypes import bytes, unicode + +jsonmod = None + +priority = ['simplejson', 'jsonlib2', 'json'] +for mod in priority: + try: + jsonmod = __import__(mod) + except ImportError: + pass + else: + break + +def dumps(o, **kwargs): + """Serialize object to JSON bytes (utf-8). + + See jsonapi.jsonmod.dumps for details on kwargs. + """ + + if 'separators' not in kwargs: + kwargs['separators'] = (',', ':') + + s = jsonmod.dumps(o, **kwargs) + + if isinstance(s, unicode): + s = s.encode('utf8') + + return s + +def loads(s, **kwargs): + """Load object from JSON bytes (utf-8). + + See jsonapi.jsonmod.loads for details on kwargs. + """ + + if str is unicode and isinstance(s, bytes): + s = s.decode('utf8') + + return jsonmod.loads(s, **kwargs) + +__all__ = ['jsonmod', 'dumps', 'loads'] + diff --git a/scripts/external_libs/zmq/utils/monitor.py b/scripts/external_libs/zmq/utils/monitor.py new file mode 100644 index 00000000..734d54b1 --- /dev/null +++ b/scripts/external_libs/zmq/utils/monitor.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +"""Module holding utility and convenience functions for zmq event monitoring.""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import struct +import zmq +from zmq.error import _check_version + +def parse_monitor_message(msg): + """decode zmq_monitor event messages. + + Parameters + ---------- + msg : list(bytes) + zmq multipart message that has arrived on a monitor PAIR socket. + + First frame is:: + + 16 bit event id + 32 bit event value + no padding + + Second frame is the endpoint as a bytestring + + Returns + ------- + event : dict + event description as dict with the keys `event`, `value`, and `endpoint`. + """ + + if len(msg) != 2 or len(msg[0]) != 6: + raise RuntimeError("Invalid event message format: %s" % msg) + event = {} + event['event'], event['value'] = struct.unpack("=hi", msg[0]) + event['endpoint'] = msg[1] + return event + +def recv_monitor_message(socket, flags=0): + """Receive and decode the given raw message from the monitoring socket and return a dict. + + Requires libzmq ≥ 4.0 + + The returned dict will have the following entries: + event : int, the event id as described in libzmq.zmq_socket_monitor + value : int, the event value associated with the event, see libzmq.zmq_socket_monitor + endpoint : string, the affected endpoint + + Parameters + ---------- + socket : zmq PAIR socket + The PAIR socket (created by other.get_monitor_socket()) on which to recv the message + flags : bitfield (int) + standard zmq recv flags + + Returns + ------- + event : dict + event description as dict with the keys `event`, `value`, and `endpoint`. + """ + _check_version((4,0), 'libzmq event API') + # will always return a list + msg = socket.recv_multipart(flags) + # 4.0-style event API + return parse_monitor_message(msg) + +__all__ = ['parse_monitor_message', 'recv_monitor_message'] diff --git a/scripts/external_libs/zmq/utils/pyversion_compat.h b/scripts/external_libs/zmq/utils/pyversion_compat.h new file mode 100644 index 00000000..fac09046 --- /dev/null +++ b/scripts/external_libs/zmq/utils/pyversion_compat.h @@ -0,0 +1,25 @@ +#include "Python.h" + +#if PY_VERSION_HEX < 0x02070000 + #define PyMemoryView_FromBuffer(info) (PyErr_SetString(PyExc_NotImplementedError, \ + "new buffer interface is not available"), (PyObject *)NULL) + #define PyMemoryView_FromObject(object) (PyErr_SetString(PyExc_NotImplementedError, \ + "new buffer interface is not available"), (PyObject *)NULL) +#endif + +#if PY_VERSION_HEX >= 0x03000000 + // for buffers + #define Py_END_OF_BUFFER ((Py_ssize_t) 0) + + #define PyObject_CheckReadBuffer(object) (0) + + #define PyBuffer_FromMemory(ptr, s) (PyErr_SetString(PyExc_NotImplementedError, \ + "old buffer interface is not available"), (PyObject *)NULL) + #define PyBuffer_FromReadWriteMemory(ptr, s) (PyErr_SetString(PyExc_NotImplementedError, \ + "old buffer interface is not available"), (PyObject *)NULL) + #define PyBuffer_FromObject(object, offset, size) (PyErr_SetString(PyExc_NotImplementedError, \ + "old buffer interface is not available"), (PyObject *)NULL) + #define PyBuffer_FromReadWriteObject(object, offset, size) (PyErr_SetString(PyExc_NotImplementedError, \ + "old buffer interface is not available"), (PyObject *)NULL) + +#endif diff --git a/scripts/external_libs/zmq/utils/sixcerpt.py b/scripts/external_libs/zmq/utils/sixcerpt.py new file mode 100644 index 00000000..5492fd59 --- /dev/null +++ b/scripts/external_libs/zmq/utils/sixcerpt.py @@ -0,0 +1,52 @@ +"""Excerpts of six.py""" + +# Copyright (C) 2010-2014 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sys + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + +if PY3: + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + + exec_("""def reraise(tp, value, tb=None): + raise tp, value, tb +""") diff --git a/scripts/external_libs/zmq/utils/strtypes.py b/scripts/external_libs/zmq/utils/strtypes.py new file mode 100644 index 00000000..548410dc --- /dev/null +++ b/scripts/external_libs/zmq/utils/strtypes.py @@ -0,0 +1,45 @@ +"""Declare basic string types unambiguously for various Python versions. + +Authors +------- +* MinRK +""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import sys + +if sys.version_info[0] >= 3: + bytes = bytes + unicode = str + basestring = (bytes, unicode) +else: + unicode = unicode + bytes = str + basestring = basestring + +def cast_bytes(s, encoding='utf8', errors='strict'): + """cast unicode or bytes to bytes""" + if isinstance(s, bytes): + return s + elif isinstance(s, unicode): + return s.encode(encoding, errors) + else: + raise TypeError("Expected unicode or bytes, got %r" % s) + +def cast_unicode(s, encoding='utf8', errors='strict'): + """cast bytes or unicode to unicode""" + if isinstance(s, bytes): + return s.decode(encoding, errors) + elif isinstance(s, unicode): + return s + else: + raise TypeError("Expected unicode or bytes, got %r" % s) + +# give short 'b' alias for cast_bytes, so that we can use fake b('stuff') +# to simulate b'stuff' +b = asbytes = cast_bytes +u = cast_unicode + +__all__ = ['asbytes', 'bytes', 'unicode', 'basestring', 'b', 'u', 'cast_bytes', 'cast_unicode'] diff --git a/scripts/external_libs/zmq/utils/win32.py b/scripts/external_libs/zmq/utils/win32.py new file mode 100644 index 00000000..ea758299 --- /dev/null +++ b/scripts/external_libs/zmq/utils/win32.py @@ -0,0 +1,132 @@ +"""Win32 compatibility utilities.""" + +#----------------------------------------------------------------------------- +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. +#----------------------------------------------------------------------------- + +import os + +# No-op implementation for other platforms. +class _allow_interrupt(object): + """Utility for fixing CTRL-C events on Windows. + + On Windows, the Python interpreter intercepts CTRL-C events in order to + translate them into ``KeyboardInterrupt`` exceptions. It (presumably) + does this by setting a flag in its "control control handler" and + checking it later at a convenient location in the interpreter. + + However, when the Python interpreter is blocked waiting for the ZMQ + poll operation to complete, it must wait for ZMQ's ``select()`` + operation to complete before translating the CTRL-C event into the + ``KeyboardInterrupt`` exception. + + The only way to fix this seems to be to add our own "console control + handler" and perform some application-defined operation that will + unblock the ZMQ polling operation in order to force ZMQ to pass control + back to the Python interpreter. + + This context manager performs all that Windows-y stuff, providing you + with a hook that is called when a CTRL-C event is intercepted. This + hook allows you to unblock your ZMQ poll operation immediately, which + will then result in the expected ``KeyboardInterrupt`` exception. + + Without this context manager, your ZMQ-based application will not + respond normally to CTRL-C events on Windows. If a CTRL-C event occurs + while blocked on ZMQ socket polling, the translation to a + ``KeyboardInterrupt`` exception will be delayed until the I/O completes + and control returns to the Python interpreter (this may never happen if + you use an infinite timeout). + + A no-op implementation is provided on non-Win32 systems to avoid the + application from having to conditionally use it. + + Example usage: + + .. sourcecode:: python + + def stop_my_application(): + # ... + + with allow_interrupt(stop_my_application): + # main polling loop. + + In a typical ZMQ application, you would use the "self pipe trick" to + send message to a ``PAIR`` socket in order to interrupt your blocking + socket polling operation. + + In a Tornado event loop, you can use the ``IOLoop.stop`` method to + unblock your I/O loop. + """ + + def __init__(self, action=None): + """Translate ``action`` into a CTRL-C handler. + + ``action`` is a callable that takes no arguments and returns no + value (returned value is ignored). It must *NEVER* raise an + exception. + + If unspecified, a no-op will be used. + """ + self._init_action(action) + + def _init_action(self, action): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return + +if os.name == 'nt': + from ctypes import WINFUNCTYPE, windll + from ctypes.wintypes import BOOL, DWORD + + kernel32 = windll.LoadLibrary('kernel32') + + # <http://msdn.microsoft.com/en-us/library/ms686016.aspx> + PHANDLER_ROUTINE = WINFUNCTYPE(BOOL, DWORD) + SetConsoleCtrlHandler = kernel32.SetConsoleCtrlHandler + SetConsoleCtrlHandler.argtypes = (PHANDLER_ROUTINE, BOOL) + SetConsoleCtrlHandler.restype = BOOL + + class allow_interrupt(_allow_interrupt): + __doc__ = _allow_interrupt.__doc__ + + def _init_action(self, action): + if action is None: + action = lambda: None + self.action = action + @PHANDLER_ROUTINE + def handle(event): + if event == 0: # CTRL_C_EVENT + action() + # Typical C implementations would return 1 to indicate that + # the event was processed and other control handlers in the + # stack should not be executed. However, that would + # prevent the Python interpreter's handler from translating + # CTRL-C to a `KeyboardInterrupt` exception, so we pretend + # that we didn't handle it. + return 0 + self.handle = handle + + def __enter__(self): + """Install the custom CTRL-C handler.""" + result = SetConsoleCtrlHandler(self.handle, 1) + if result == 0: + # Have standard library automatically call `GetLastError()` and + # `FormatMessage()` into a nice exception object :-) + raise WindowsError() + + def __exit__(self, *args): + """Remove the custom CTRL-C handler.""" + result = SetConsoleCtrlHandler(self.handle, 0) + if result == 0: + # Have standard library automatically call `GetLastError()` and + # `FormatMessage()` into a nice exception object :-) + raise WindowsError() +else: + class allow_interrupt(_allow_interrupt): + __doc__ = _allow_interrupt.__doc__ + pass diff --git a/scripts/external_libs/zmq/utils/z85.py b/scripts/external_libs/zmq/utils/z85.py new file mode 100644 index 00000000..1bb1784e --- /dev/null +++ b/scripts/external_libs/zmq/utils/z85.py @@ -0,0 +1,56 @@ +"""Python implementation of Z85 85-bit encoding + +Z85 encoding is a plaintext encoding for a bytestring interpreted as 32bit integers. +Since the chunks are 32bit, a bytestring must be a multiple of 4 bytes. +See ZMQ RFC 32 for details. + + +""" + +# Copyright (C) PyZMQ Developers +# Distributed under the terms of the Modified BSD License. + +import sys +import struct + +PY3 = sys.version_info[0] >= 3 +# Z85CHARS is the base 85 symbol table +Z85CHARS = b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#" +# Z85MAP maps integers in [0,84] to the appropriate character in Z85CHARS +Z85MAP = dict([(c, idx) for idx, c in enumerate(Z85CHARS)]) + +_85s = [ 85**i for i in range(5) ][::-1] + +def encode(rawbytes): + """encode raw bytes into Z85""" + # Accepts only byte arrays bounded to 4 bytes + if len(rawbytes) % 4: + raise ValueError("length must be multiple of 4, not %i" % len(rawbytes)) + + nvalues = len(rawbytes) / 4 + + values = struct.unpack('>%dI' % nvalues, rawbytes) + encoded = [] + for v in values: + for offset in _85s: + encoded.append(Z85CHARS[(v // offset) % 85]) + + # In Python 3, encoded is a list of integers (obviously?!) + if PY3: + return bytes(encoded) + else: + return b''.join(encoded) + +def decode(z85bytes): + """decode Z85 bytes to raw bytes""" + if len(z85bytes) % 5: + raise ValueError("Z85 length must be multiple of 5, not %i" % len(z85bytes)) + + nvalues = len(z85bytes) / 5 + values = [] + for i in range(0, len(z85bytes), 5): + value = 0 + for j, offset in enumerate(_85s): + value += Z85MAP[z85bytes[i+j]] * offset + values.append(value) + return struct.pack('>%dI' % nvalues, *values) diff --git a/scripts/external_libs/zmq/utils/zmq_compat.h b/scripts/external_libs/zmq/utils/zmq_compat.h new file mode 100644 index 00000000..81c57b69 --- /dev/null +++ b/scripts/external_libs/zmq/utils/zmq_compat.h @@ -0,0 +1,80 @@ +//----------------------------------------------------------------------------- +// Copyright (c) 2010 Brian Granger, Min Ragan-Kelley +// +// Distributed under the terms of the New BSD License. The full license is in +// the file COPYING.BSD, distributed as part of this software. +//----------------------------------------------------------------------------- + +#if defined(_MSC_VER) +#define pyzmq_int64_t __int64 +#else +#include <stdint.h> +#define pyzmq_int64_t int64_t +#endif + + +#include "zmq.h" +// version compatibility for constants: +#include "zmq_constants.h" + +#define _missing (-1) + + +// define fd type (from libzmq's fd.hpp) +#ifdef _WIN32 + #ifdef _MSC_VER && _MSC_VER <= 1400 + #define ZMQ_FD_T UINT_PTR + #else + #define ZMQ_FD_T SOCKET + #endif +#else + #define ZMQ_FD_T int +#endif + +// use unambiguous aliases for zmq_send/recv functions + +#if ZMQ_VERSION_MAJOR >= 4 +// nothing to remove +#else + #define zmq_curve_keypair(z85_public_key, z85_secret_key) _missing +#endif + +#if ZMQ_VERSION_MAJOR >= 4 && ZMQ_VERSION_MINOR >= 1 +// nothing to remove +#else + #define zmq_msg_gets(msg, prop) _missing + #define zmq_has(capability) _missing +#endif + +#if ZMQ_VERSION_MAJOR >= 3 + #define zmq_sendbuf zmq_send + #define zmq_recvbuf zmq_recv + + // 3.x deprecations - these symbols haven't been removed, + // but let's protect against their planned removal + #define zmq_device(device_type, isocket, osocket) _missing + #define zmq_init(io_threads) ((void*)NULL) + #define zmq_term zmq_ctx_destroy +#else + #define zmq_ctx_set(ctx, opt, val) _missing + #define zmq_ctx_get(ctx, opt) _missing + #define zmq_ctx_destroy zmq_term + #define zmq_ctx_new() ((void*)NULL) + + #define zmq_proxy(a,b,c) _missing + + #define zmq_disconnect(s, addr) _missing + #define zmq_unbind(s, addr) _missing + + #define zmq_msg_more(msg) _missing + #define zmq_msg_get(msg, opt) _missing + #define zmq_msg_set(msg, opt, val) _missing + #define zmq_msg_send(msg, s, flags) zmq_send(s, msg, flags) + #define zmq_msg_recv(msg, s, flags) zmq_recv(s, msg, flags) + + #define zmq_sendbuf(s, buf, len, flags) _missing + #define zmq_recvbuf(s, buf, len, flags) _missing + + #define zmq_socket_monitor(s, addr, flags) _missing + +#endif diff --git a/scripts/external_libs/zmq/utils/zmq_constants.h b/scripts/external_libs/zmq/utils/zmq_constants.h new file mode 100644 index 00000000..97683022 --- /dev/null +++ b/scripts/external_libs/zmq/utils/zmq_constants.h @@ -0,0 +1,622 @@ +#ifndef _PYZMQ_CONSTANT_DEFS +#define _PYZMQ_CONSTANT_DEFS + +#define _PYZMQ_UNDEFINED (-9999) +#ifndef ZMQ_VERSION + #define ZMQ_VERSION (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_VERSION_MAJOR + #define ZMQ_VERSION_MAJOR (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_VERSION_MINOR + #define ZMQ_VERSION_MINOR (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_VERSION_PATCH + #define ZMQ_VERSION_PATCH (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_NOBLOCK + #define ZMQ_NOBLOCK (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_DONTWAIT + #define ZMQ_DONTWAIT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_POLLIN + #define ZMQ_POLLIN (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_POLLOUT + #define ZMQ_POLLOUT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_POLLERR + #define ZMQ_POLLERR (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SNDMORE + #define ZMQ_SNDMORE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_STREAMER + #define ZMQ_STREAMER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_FORWARDER + #define ZMQ_FORWARDER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_QUEUE + #define ZMQ_QUEUE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_IO_THREADS_DFLT + #define ZMQ_IO_THREADS_DFLT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_MAX_SOCKETS_DFLT + #define ZMQ_MAX_SOCKETS_DFLT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_POLLITEMS_DFLT + #define ZMQ_POLLITEMS_DFLT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_THREAD_PRIORITY_DFLT + #define ZMQ_THREAD_PRIORITY_DFLT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_THREAD_SCHED_POLICY_DFLT + #define ZMQ_THREAD_SCHED_POLICY_DFLT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_PAIR + #define ZMQ_PAIR (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_PUB + #define ZMQ_PUB (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SUB + #define ZMQ_SUB (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_REQ + #define ZMQ_REQ (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_REP + #define ZMQ_REP (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_DEALER + #define ZMQ_DEALER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_ROUTER + #define ZMQ_ROUTER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_XREQ + #define ZMQ_XREQ (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_XREP + #define ZMQ_XREP (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_PULL + #define ZMQ_PULL (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_PUSH + #define ZMQ_PUSH (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_XPUB + #define ZMQ_XPUB (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_XSUB + #define ZMQ_XSUB (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_UPSTREAM + #define ZMQ_UPSTREAM (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_DOWNSTREAM + #define ZMQ_DOWNSTREAM (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_STREAM + #define ZMQ_STREAM (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_CONNECTED + #define ZMQ_EVENT_CONNECTED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_CONNECT_DELAYED + #define ZMQ_EVENT_CONNECT_DELAYED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_CONNECT_RETRIED + #define ZMQ_EVENT_CONNECT_RETRIED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_LISTENING + #define ZMQ_EVENT_LISTENING (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_BIND_FAILED + #define ZMQ_EVENT_BIND_FAILED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_ACCEPTED + #define ZMQ_EVENT_ACCEPTED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_ACCEPT_FAILED + #define ZMQ_EVENT_ACCEPT_FAILED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_CLOSED + #define ZMQ_EVENT_CLOSED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_CLOSE_FAILED + #define ZMQ_EVENT_CLOSE_FAILED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_DISCONNECTED + #define ZMQ_EVENT_DISCONNECTED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_ALL + #define ZMQ_EVENT_ALL (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENT_MONITOR_STOPPED + #define ZMQ_EVENT_MONITOR_STOPPED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_NULL + #define ZMQ_NULL (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_PLAIN + #define ZMQ_PLAIN (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_CURVE + #define ZMQ_CURVE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_GSSAPI + #define ZMQ_GSSAPI (_PYZMQ_UNDEFINED) +#endif + +#ifndef EAGAIN + #define EAGAIN (_PYZMQ_UNDEFINED) +#endif + +#ifndef EINVAL + #define EINVAL (_PYZMQ_UNDEFINED) +#endif + +#ifndef EFAULT + #define EFAULT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ENOMEM + #define ENOMEM (_PYZMQ_UNDEFINED) +#endif + +#ifndef ENODEV + #define ENODEV (_PYZMQ_UNDEFINED) +#endif + +#ifndef EMSGSIZE + #define EMSGSIZE (_PYZMQ_UNDEFINED) +#endif + +#ifndef EAFNOSUPPORT + #define EAFNOSUPPORT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ENETUNREACH + #define ENETUNREACH (_PYZMQ_UNDEFINED) +#endif + +#ifndef ECONNABORTED + #define ECONNABORTED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ECONNRESET + #define ECONNRESET (_PYZMQ_UNDEFINED) +#endif + +#ifndef ENOTCONN + #define ENOTCONN (_PYZMQ_UNDEFINED) +#endif + +#ifndef ETIMEDOUT + #define ETIMEDOUT (_PYZMQ_UNDEFINED) +#endif + +#ifndef EHOSTUNREACH + #define EHOSTUNREACH (_PYZMQ_UNDEFINED) +#endif + +#ifndef ENETRESET + #define ENETRESET (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_HAUSNUMERO + #define ZMQ_HAUSNUMERO (_PYZMQ_UNDEFINED) +#endif + +#ifndef ENOTSUP + #define ENOTSUP (_PYZMQ_UNDEFINED) +#endif + +#ifndef EPROTONOSUPPORT + #define EPROTONOSUPPORT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ENOBUFS + #define ENOBUFS (_PYZMQ_UNDEFINED) +#endif + +#ifndef ENETDOWN + #define ENETDOWN (_PYZMQ_UNDEFINED) +#endif + +#ifndef EADDRINUSE + #define EADDRINUSE (_PYZMQ_UNDEFINED) +#endif + +#ifndef EADDRNOTAVAIL + #define EADDRNOTAVAIL (_PYZMQ_UNDEFINED) +#endif + +#ifndef ECONNREFUSED + #define ECONNREFUSED (_PYZMQ_UNDEFINED) +#endif + +#ifndef EINPROGRESS + #define EINPROGRESS (_PYZMQ_UNDEFINED) +#endif + +#ifndef ENOTSOCK + #define ENOTSOCK (_PYZMQ_UNDEFINED) +#endif + +#ifndef EFSM + #define EFSM (_PYZMQ_UNDEFINED) +#endif + +#ifndef ENOCOMPATPROTO + #define ENOCOMPATPROTO (_PYZMQ_UNDEFINED) +#endif + +#ifndef ETERM + #define ETERM (_PYZMQ_UNDEFINED) +#endif + +#ifndef EMTHREAD + #define EMTHREAD (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_IO_THREADS + #define ZMQ_IO_THREADS (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_MAX_SOCKETS + #define ZMQ_MAX_SOCKETS (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SOCKET_LIMIT + #define ZMQ_SOCKET_LIMIT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_THREAD_PRIORITY + #define ZMQ_THREAD_PRIORITY (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_THREAD_SCHED_POLICY + #define ZMQ_THREAD_SCHED_POLICY (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_IDENTITY + #define ZMQ_IDENTITY (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SUBSCRIBE + #define ZMQ_SUBSCRIBE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_UNSUBSCRIBE + #define ZMQ_UNSUBSCRIBE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_LAST_ENDPOINT + #define ZMQ_LAST_ENDPOINT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_TCP_ACCEPT_FILTER + #define ZMQ_TCP_ACCEPT_FILTER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_PLAIN_USERNAME + #define ZMQ_PLAIN_USERNAME (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_PLAIN_PASSWORD + #define ZMQ_PLAIN_PASSWORD (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_CURVE_PUBLICKEY + #define ZMQ_CURVE_PUBLICKEY (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_CURVE_SECRETKEY + #define ZMQ_CURVE_SECRETKEY (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_CURVE_SERVERKEY + #define ZMQ_CURVE_SERVERKEY (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_ZAP_DOMAIN + #define ZMQ_ZAP_DOMAIN (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_CONNECT_RID + #define ZMQ_CONNECT_RID (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_GSSAPI_PRINCIPAL + #define ZMQ_GSSAPI_PRINCIPAL (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_GSSAPI_SERVICE_PRINCIPAL + #define ZMQ_GSSAPI_SERVICE_PRINCIPAL (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SOCKS_PROXY + #define ZMQ_SOCKS_PROXY (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_FD + #define ZMQ_FD (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_IDENTITY_FD + #define ZMQ_IDENTITY_FD (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_RECONNECT_IVL_MAX + #define ZMQ_RECONNECT_IVL_MAX (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SNDTIMEO + #define ZMQ_SNDTIMEO (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_RCVTIMEO + #define ZMQ_RCVTIMEO (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SNDHWM + #define ZMQ_SNDHWM (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_RCVHWM + #define ZMQ_RCVHWM (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_MULTICAST_HOPS + #define ZMQ_MULTICAST_HOPS (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_IPV4ONLY + #define ZMQ_IPV4ONLY (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_ROUTER_BEHAVIOR + #define ZMQ_ROUTER_BEHAVIOR (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_TCP_KEEPALIVE + #define ZMQ_TCP_KEEPALIVE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_TCP_KEEPALIVE_CNT + #define ZMQ_TCP_KEEPALIVE_CNT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_TCP_KEEPALIVE_IDLE + #define ZMQ_TCP_KEEPALIVE_IDLE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_TCP_KEEPALIVE_INTVL + #define ZMQ_TCP_KEEPALIVE_INTVL (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_DELAY_ATTACH_ON_CONNECT + #define ZMQ_DELAY_ATTACH_ON_CONNECT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_XPUB_VERBOSE + #define ZMQ_XPUB_VERBOSE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_EVENTS + #define ZMQ_EVENTS (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_TYPE + #define ZMQ_TYPE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_LINGER + #define ZMQ_LINGER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_RECONNECT_IVL + #define ZMQ_RECONNECT_IVL (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_BACKLOG + #define ZMQ_BACKLOG (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_ROUTER_MANDATORY + #define ZMQ_ROUTER_MANDATORY (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_FAIL_UNROUTABLE + #define ZMQ_FAIL_UNROUTABLE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_ROUTER_RAW + #define ZMQ_ROUTER_RAW (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_IMMEDIATE + #define ZMQ_IMMEDIATE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_IPV6 + #define ZMQ_IPV6 (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_MECHANISM + #define ZMQ_MECHANISM (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_PLAIN_SERVER + #define ZMQ_PLAIN_SERVER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_CURVE_SERVER + #define ZMQ_CURVE_SERVER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_PROBE_ROUTER + #define ZMQ_PROBE_ROUTER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_REQ_RELAXED + #define ZMQ_REQ_RELAXED (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_REQ_CORRELATE + #define ZMQ_REQ_CORRELATE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_CONFLATE + #define ZMQ_CONFLATE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_ROUTER_HANDOVER + #define ZMQ_ROUTER_HANDOVER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_TOS + #define ZMQ_TOS (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_IPC_FILTER_PID + #define ZMQ_IPC_FILTER_PID (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_IPC_FILTER_UID + #define ZMQ_IPC_FILTER_UID (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_IPC_FILTER_GID + #define ZMQ_IPC_FILTER_GID (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_GSSAPI_SERVER + #define ZMQ_GSSAPI_SERVER (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_GSSAPI_PLAINTEXT + #define ZMQ_GSSAPI_PLAINTEXT (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_HANDSHAKE_IVL + #define ZMQ_HANDSHAKE_IVL (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_XPUB_NODROP + #define ZMQ_XPUB_NODROP (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_AFFINITY + #define ZMQ_AFFINITY (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_MAXMSGSIZE + #define ZMQ_MAXMSGSIZE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_HWM + #define ZMQ_HWM (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SWAP + #define ZMQ_SWAP (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_MCAST_LOOP + #define ZMQ_MCAST_LOOP (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_RECOVERY_IVL_MSEC + #define ZMQ_RECOVERY_IVL_MSEC (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_RATE + #define ZMQ_RATE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_RECOVERY_IVL + #define ZMQ_RECOVERY_IVL (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SNDBUF + #define ZMQ_SNDBUF (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_RCVBUF + #define ZMQ_RCVBUF (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_RCVMORE + #define ZMQ_RCVMORE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_MORE + #define ZMQ_MORE (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SRCFD + #define ZMQ_SRCFD (_PYZMQ_UNDEFINED) +#endif + +#ifndef ZMQ_SHARED + #define ZMQ_SHARED (_PYZMQ_UNDEFINED) +#endif + + +#endif // ifndef _PYZMQ_CONSTANT_DEFS |