Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • gmr/queries
1 result
Show changes
Commits on Source (11)
Hide whitespace changes
Inline Side-by-side
LICENSE
View file @ 90ab6006
Copyright (c) 2014 - 2016 Gavin M. Roy
Copyright (c) 2014 - 2018 Gavin M. Roy
All rights reserved.
 
Redistribution and use in source and binary forms, with or without modification,
README.rst
View file @ 90ab6006
@@ -21,7 +21,7 @@ Key features include:
- Ability to directly access psycopg2 ``connection`` and ``cursor`` objects
- Internal connection pooling
 
|Version| |Status| |Coverage| |CodeClimate| |License| |PythonVersions|
|Version| |Status| |Coverage| |License|
 
Documentation
-------------
@@ -181,9 +181,3 @@ main GitHub repository of Queries as tags prior to version 1.2.0.
 
.. |License| image:: https://img.shields.io/github/license/gmr/queries.svg?
:target: https://github.com/gmr/queries
.. |CodeClimate| image:: https://img.shields.io/codeclimate/github/gmr/queries.svg?
:target: https://codeclimate.com/github/gmr/queries
.. |PythonVersions| image:: https://img.shields.io/pypi/pyversions/queries.svg?
:target: https://github.com/gmr/queries
docs/conf.py
View file @ 90ab6006
# -*- coding: utf-8 -*-
#
# Queries documentation build configuration file, created by
# sphinx-quickstart on Fri Apr 25 10:36:39 2014.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import datetime
import sys
sys.path.insert(0, '../')
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
 
# -- General configuration ------------------------------------------------
sys.path.insert(0, '../')
 
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
import queries
 
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.intersphinx',
'sphinx.ext.viewcode',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'Queries'
copyright = u'2014 - 2016, Gavin M. Roy'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
import queries
project = 'Queries'
copyright = '2014 - {}, Gavin M. Roy'.format(
datetime.date.today().strftime('%Y'))
release = queries.__version__
version = '.'.join(release.split('.')[0:1])
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'default'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'Queriesdoc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_elements = {}
latex_documents = [
('index', 'Queries.tex', u'Queries Documentation',
u'Gavin M. Roy', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('index', 'queries', u'Queries Documentation',
[u'Gavin M. Roy'], 1)
]
# If true, show URL addresses after external links.
#man_show_urls = False
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'Queries', u'Queries Documentation',
u'Gavin M. Roy', 'Queries', 'PostgreSQL Simplified',
'Miscellaneous'),
]
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []
# If false, no module index is generated.
#texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'psycopg2': ('http://initd.org/psycopg/docs/', None),
'tornado': ('http://www.tornadoweb.org/en/stable', None)}
docs/history.rst
View file @ 90ab6006
Version History
===============
 
2.0.0 2018-01-23
2.0.0 2018-01-29
-----------------
- REMOVED support for Python 2.6
- FIXED CPU Pegging bug: Cleanup IOLoop and internal stack in ``TornadoSession``
on connection error. In the case of a connection error, the failure to do this
caused CPU to peg @ 100% utilization looping on a non-existent file descriptor.
Thanks to `cknave <https://github.com/cknave>`_ for his work on identifying the
issue, proposing a fix, and writing a working test case.
- FIXED CPU Pegging bug: Cleanup IOLoop and internal stack in ``TornadoSession`` on connection error. In the case of a connection error, the failure to do this caused CPU to peg @ 100% utilization looping on a non-existent file descriptor. Thanks to `cknave <https://github.com/cknave>`_ for his work on identifying the issue, proposing a fix, and writing a working test case.
- Move the integration tests to use a local docker development environment
- Added new methods ``queries.pool.Pool.report`` and
``queries.pool.PoolManager.Report` for reporting pool status.
- Added new methods to ``queries.pool.Pool`` for returning a list of
busy, closed, executing, and locked connections.
- Added new methods ``queries.pool.Pool.report`` and ``queries.pool.PoolManager.Report`` for reporting pool status.
- Added new methods to ``queries.pool.Pool`` for returning a list of busy, closed, executing, and locked connections.
 
1.10.4 2018-01-10
-----------------
@@ -36,9 +30,9 @@ Version History
 
1.10.0 2017-09-27
-----------------
- Free when tornado_session.Result is ``__del__'d`` without ``free`` being called.
- Free when tornado_session.Result is ``__del__``'d without ``free`` being called.
- Auto-clean the pool after Results.free TTL+1 in tornado_session.TornadoSession
- Dont raise NotImplementedError in Results.free for synchronous use, just treat as a noop
- Don't raise NotImplementedError in Results.free for synchronous use, just treat as a noop
 
1.9.1 2016-10-25
----------------
docs/index.rst
View file @ 90ab6006
@@ -3,7 +3,7 @@ Queries: PostgreSQL Simplified
*Queries* is a BSD licensed opinionated wrapper of the psycopg2_ library for
interacting with PostgreSQL.
 
|Version| |License| |PythonVersions|
|Version| |License|
 
The popular psycopg2_ package is a full-featured python client. Unfortunately
as a developer, you're often repeating the same steps to get started with your
@@ -53,8 +53,6 @@ Source
------
Queries source is available on Github at `https://github.com/gmr/queries <https://github.com/gmr/queries>`_
 
|Status|
Inspiration
-----------
Queries is inspired by `Kenneth Reitz's <https://github.com/kennethreitz/>`_ awesome
@@ -81,6 +79,3 @@ Indices and tables
 
.. |License| image:: https://img.shields.io/github/license/gmr/queries.svg?
:target: https://github.com/gmr/queries
.. |PythonVersions| image:: https://img.shields.io/pypi/pyversions/queries.svg?
:target: https://github.com/gmr/queries
examples/simple-tornado.py
View file @ 90ab6006
import datetime
import logging
from queries import pool
import queries
from tornado import gen, ioloop, web
 
 
class ExampleHandler(web.RequestHandler):
 
def initialize(self):
self.session = queries.TornadoSession()
SQL = 'SELECT * FROM pg_stat_activity'
 
@gen.coroutine
def get(self):
try:
result = yield self.session.query('SELECT * FROM names')
result = yield self.application.session.query(self.SQL)
except queries.OperationalError as error:
logging.error('Error connecting to the database: %s', error)
raise web.HTTPError(503)
 
self.finish({'data': result.items()})
rows = []
for row in result.items():
row = dict([(k, v.isoformat()
if isinstance(v, datetime.datetime) else v)
for k, v in row.items()])
rows.append(row)
result.free()
self.finish({'pg_stat_activity': rows})
class ReportHandler(web.RequestHandler):
 
@gen.coroutine
def get(self):
self.finish(pool.PoolManager.report())
 
application = web.Application([
(r'/', ExampleHandler),
])
 
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
application.listen(8888)
logging.basicConfig(level=logging.DEBUG)
application = web.Application([
(r'/', ExampleHandler),
(r'/report', ReportHandler)
], debug=True)
application.session = queries.TornadoSession()
application.listen(8000)
ioloop.IOLoop.instance().start()
queries/pool.py
View file @ 90ab6006
@@ -53,7 +53,7 @@ class Connection(object):
:rtype: bool
 
"""
return self.handle.closed
return self.handle.closed != 0
 
@property
def busy(self):
@@ -181,7 +181,8 @@ class Pool(object):
:rtype: list
 
"""
return [c for c in self.connections if c.busy and not c.closed]
return [c for c in self.connections.values()
if c.busy and not c.closed]
 
def clean(self):
"""Clean the pool by removing any closed connections and if the pool's
@@ -231,7 +232,7 @@ class Pool(object):
:rtype: list
 
"""
return [c for c in self.connections if c.executing]
return [c for c in self.connections.values() if c.executing]
 
def free(self, connection):
"""Free the connection from use by the session that was using it.
@@ -365,14 +366,13 @@ class Pool(object):
'connections': {
'busy': len(self.busy_connections),
'closed': len(self.closed_connections),
'executing': len([c for c in self.connections
if c.executing()]),
'executing': len(self.executing_connections),
'idle': len(self.idle_connections),
'locked': len(self.busy_connections)
},
'exceptions': sum([c.exceptions
for c in self.connections.values()]),
'executions': sum([c.executed
'executions': sum([c.executions
for c in self.connections.values()]),
'full': self.is_full,
'idle': {
queries/tornado_session.py
View file @ 90ab6006
@@ -295,7 +295,7 @@ class TornadoSession(session.Session):
 
"""
if cf.exception():
self._cleanup_fd(fd)
self._cleanup_fd(fd, True)
future.set_exception(cf.exception())
 
else:
@@ -372,11 +372,13 @@ class TornadoSession(session.Session):
def completed(qf):
"""Invoked by the IOLoop when the future has completed"""
if qf.exception():
self._incr_exceptions(conn)
err = qf.exception()
LOGGER.debug('Cleaning cursor due to exception: %r', err)
self._exec_cleanup(cursor, conn.fileno())
future.set_exception(err)
else:
self._incr_executions(conn)
value = Results(cursor, self._exec_cleanup, conn.fileno())
future.set_result(value)
 
@@ -428,7 +430,7 @@ class TornadoSession(session.Session):
self._ioloop.time() + self._pool_idle_ttl + 1,
self._pool_manager.clean, self.pid)
 
def _cleanup_fd(self, fd):
def _cleanup_fd(self, fd, close=False):
"""Ensure the socket socket is removed from the IOLoop, the
connection stack, and futures stack.
 
@@ -441,11 +443,28 @@ class TornadoSession(session.Session):
self._pool_manager.free(self.pid, self._connections[fd])
except pool.ConnectionNotFoundError:
pass
self._connections[fd].close()
if close:
self._connections[fd].close()
del self._connections[fd]
if fd in self._futures:
del self._futures[fd]
 
def _incr_exceptions(self, conn):
"""Increment the number of exceptions for the current connection.
:param psycopg2.extensions.connection conn: the psycopg2 connection
"""
self._pool_manager.get_connection(self.pid, conn).exceptions += 1
def _incr_executions(self, conn):
"""Increment the number of executions for the current connection.
:param psycopg2.extensions.connection conn: the psycopg2 connection
"""
self._pool_manager.get_connection(self.pid, conn).executions += 1
def _on_io_events(self, fd=None, _events=None):
"""Invoked by Tornado's IOLoop when there are events for the fd
 
requires/testing.txt
View file @ 90ab6006
@@ -2,5 +2,4 @@ coverage
flake8
mock
nose
codecov
tornado
setup.py
View file @ 90ab6006
@@ -19,6 +19,7 @@ classifiers = ['Development Status :: 5 - Production/Stable',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: Implementation :: CPython',