CONTRIBUTORS.txt

@@ -141,3 +141,10 @@ Order doesn't matter (not that much, at least ;)
 * Ahirnish Pareek, 'keyword-arg-before-var-arg' check
 * Guillaume Peillex: contributor.
+
+* Daniel Miller: contributor.
+
+* Bryce Guinta: contributor
+
+* Martin Bašti: contributor
+  Added new check for shallow copy of os.environ

ChangeLog

@@ -4,6 +4,28 @@ Pylint's ChangeLog
 What's New in Pylint 1.8?
 =========================
+    * Respect disable=... in config file when running with --py3k.
+
+    * New warning `shallow-copy-environ` added
+
+      Shallow copy of os.environ doesn't work as people may expect. os.environ
+      is not a dict object but rather a proxy object, so any changes made
+      on copy may have unexpected effects on os.environ
+
+      Instead of copy.copy(os.environ) method os.environ.copy() should be
+      used.
+
+      See https://bugs.python.org/issue15373 for details.
+
+      Close #1301
+
+    * Do not display no-absolute-import warning multiple times per file.
+
+    * `trailing-comma-tuple` refactor check now extends to assignment with
+       more than one element (such as lists)
+
+      Close #1713
+
     * Fixing u'' string in superfluous-parens message 
       Close #1420
@@ -130,10 +152,17 @@ What's New in Pylint 1.8?
       mixing ``Args`` and ``Keyword Args`` in Google docstring.
       Close #1409
+     * Fix ``missing-docstring`` false negatives when modules, classes, or methods
+     consist of compound statements that exceed the ``docstring-min-length``
+
     * Fix ``useless-else-on-loop`` false positives when break statements are
       deeply nested inside loop.
       Close #1661
+    * Fix no ``wrong-import-order`` message emitted on ordering of first and third party
+      libraries. With this fix, pylint distinguishes third and first party
+      modules when checking import order.
+      Close #1702
 What's New in Pylint 1.7.1?
 =========================

doc/development_guide/contribute.rst

@@ -79,7 +79,11 @@ your patch gets accepted.
         python -m tox -e py27 -- -k  \*func\*
 - Add a short entry to the ChangeLog describing the change, except for internal
-  implementation only changes
+  implementation only changes. Not usually required, but for changes other than small
+  bugs we also add a couple of sentences in the release document for that release,
+  (`What's New` section)
+
+- Add yourself to the `CONTRIBUTORS` file, if you are not already there.
 - Write a comprehensive commit message

doc/whatsnew/1.8.rst

@@ -14,6 +14,29 @@ Summary -- Release highlights
 New checkers
 ============
+* A new check was added, ``shallow-copy-environ``.
+
+  This warning message is emitted when shallow copy of os.environ is created.
+  Shallow copy of os.environ doesn't work as people may expect. os.environ
+  is not a dict object but rather a proxy object, so any changes made
+  on copy may have unexpected effects on os.environ
+
+  Instead of copy.copy(os.environ) method os.environ.copy() should be used.
+
+  See https://bugs.python.org/issue15373 for details.
+
+  .. code-block:: python
+
+     import copy
+     import os
+     wrong_env_copy = copy.copy(os.environ)  # will emit pylint warning
+     wrong_env_copy['ENV_VAR'] = 'new_value'  # changes os.environ
+     assert os.environ['ENV_VAR'] == 'new_value'
+
+     good_env_copy = os.environ.copy()  # the right way
+     good_env_copy['ENV_VAR'] = 'different_value'  # doesn't change os.environ
+     assert os.environ['ENV_VAR'] == 'new_value'
+
 * A new check was added, ``keyword-arg-before-vararg``.
   This warning message is emitted when a function is defined with a keyword
@@ -323,3 +346,14 @@ Other Changes
 * Fix of false positive ``useless-else-on-loop`` message when break statements
   are deeply nested inside loop.
+
+* The Python 3 porting checker no longer emits multiple `no-absolute-import` per file.
+
+* The Python 3 porting checker respects disabled checkers found in the config file.
+
+* Modules, classes, or methods consist of compound statements that exceed the ``docstring-min-length``
+  are now correctly emitting `missing-docstring`
+
+* Fix no ``wrong-import-order`` message emitted on ordering of first and third party libraries.
+  With this fix, pylint distinguishes first and third party modules when checking
+  import order.

pylint/checkers/base.py

@@ -35,6 +35,7 @@ from pylint import exceptions
 from pylint import interfaces
 from pylint.checkers import utils
 from pylint import reporters
+from pylint.checkers.utils import get_node_last_lineno
 from pylint.reporters.ureports import nodes as reporter_nodes
 import pylint.utils as lint_utils
@@ -1604,10 +1605,7 @@ class DocStringChecker(_BasicChecker):
         if docstring is None:
             if not report_missing:
                 return
-            if node.body:
-                lines = node.body[-1].lineno - node.body[0].lineno + 1
-            else:
-                lines = 0
+            lines = get_node_last_lineno(node) - node.lineno
             if node_type == 'module' and not lines:
                 # If the module has no body, there's no reason

pylint/checkers/imports.py

+# -*- coding: utf-8 -*-
 # Copyright (c) 2006-2015 LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr>
 # Copyright (c) 2012-2014 Google, Inc.
 # Copyright (c) 2014-2016 Claudiu Popa <pcmanticore@gmail.com>
@@ -563,10 +564,14 @@ class ImportsChecker(BaseChecker):
         Imports must follow this order: standard, 3rd party, local
         """
-        extern_imports = []
-        local_imports = []
         std_imports = []
-        extern_not_ignored = []
+        third_party_imports = []
+        first_party_imports = []
+        # need of a list that holds third or first party ordered import
+        external_imports = []
+        local_imports = []
+        third_party_not_ignored = []
+        first_party_not_ignored = []
         local_not_ignored = []
         isort_obj = isort.SortImports(
             file_contents='', known_third_party=self.config.known_third_party,
@@ -581,29 +586,42 @@ class ImportsChecker(BaseChecker):
             ignore_for_import_order = not self.linter.is_message_enabled('wrong-import-order',
                                                                          node.fromlineno)
             import_category = isort_obj.place_module(package)
+            node_and_package_import = (node, package)
             if import_category in ('FUTURE', 'STDLIB'):
-                std_imports.append((node, package))
-                wrong_import = extern_not_ignored or local_not_ignored
+                std_imports.append(node_and_package_import)
+                wrong_import = (third_party_not_ignored or first_party_not_ignored
+                                or local_not_ignored)
                 if self._is_fallback_import(node, wrong_import):
                     continue
                 if wrong_import and not nested:
                     self.add_message('wrong-import-order', node=node,
                                      args=('standard import "%s"' % node.as_string(),
                                            '"%s"' % wrong_import[0][0].as_string()))
-            elif import_category in ('FIRSTPARTY', 'THIRDPARTY'):
-                extern_imports.append((node, package))
+            elif import_category == 'THIRDPARTY':
+                third_party_imports.append(node_and_package_import)
+                external_imports.append(node_and_package_import)
+                if not nested and not ignore_for_import_order:
+                    third_party_not_ignored.append(node_and_package_import)
+                wrong_import = first_party_not_ignored or local_not_ignored
+                if wrong_import and not nested:
+                    self.add_message('wrong-import-order', node=node,
+                                     args=('third party import "%s"' % node.as_string(),
+                                           '"%s"' % wrong_import[0][0].as_string()))
+            elif import_category == 'FIRSTPARTY':
+                first_party_imports.append(node_and_package_import)
+                external_imports.append(node_and_package_import)
                 if not nested and not ignore_for_import_order:
-                    extern_not_ignored.append((node, package))
+                    first_party_not_ignored.append(node_and_package_import)
                 wrong_import = local_not_ignored
                 if wrong_import and not nested:
                     self.add_message('wrong-import-order', node=node,
-                                     args=('external import "%s"' % node.as_string(),
+                                     args=('first party import "%s"' % node.as_string(),
                                            '"%s"' % wrong_import[0][0].as_string()))
             elif import_category == 'LOCALFOLDER':
                 local_imports.append((node, package))
                 if not nested and not ignore_for_import_order:
                     local_not_ignored.append((node, package))
-        return std_imports, extern_imports, local_imports
+        return std_imports, external_imports, local_imports
     def _get_imported_module(self, importnode, modname):
         try:

pylint/checkers/python3.py

@@ -650,6 +650,7 @@ class Python3Checker(checkers.BaseChecker):
             if not self._future_absolute_import:
                 if self.linter.is_message_enabled('no-absolute-import'):
                     self.add_message('no-absolute-import', node=node)
+                    self._future_absolute_import = True
             if not _is_conditional_import(node) and not node.level:
                 self._warn_if_deprecated(node, node.modname, {x[0] for x in node.names})
@@ -660,7 +661,9 @@ class Python3Checker(checkers.BaseChecker):
     def visit_import(self, node):
         if not self._future_absolute_import:
-            self.add_message('no-absolute-import', node=node)
+            if self.linter.is_message_enabled('no-absolute-import'):
+                self.add_message('no-absolute-import', node=node)
+                self._future_absolute_import = True
         if not _is_conditional_import(node):
             for name, _ in node.names:
                 self._warn_if_deprecated(node, name, None)

pylint/checkers/refactoring.py

@@ -271,28 +271,10 @@ class RefactoringChecker(checkers.BaseTokenChecker):
                 # tokens[index][2] is the actual position and also is
                 # reported by IronPython.
                 self._elifs.extend([tokens[index][2], tokens[index+1][2]])
-            elif six.PY3 and token.exact_type == tokenize.COMMA:
-                self._check_one_element_trailing_comma_tuple(tokens, token, index)
-
-    def _check_one_element_trailing_comma_tuple(self, tokens, token, index):
-        left_tokens = itertools.islice(tokens, index + 1, None)
-        same_line_remaining_tokens = list(itertools.takewhile(
-            lambda other_token, _token=token: other_token.start[0] == _token.start[0],
-            left_tokens
-        ))
-        is_last_element = all(
-            other_token.type in (tokenize.NEWLINE, tokenize.COMMENT)
-            for other_token in same_line_remaining_tokens
-        )
-
-        if not same_line_remaining_tokens or not is_last_element:
-            return
-
-        assign_token = tokens[index-2:index-1]
-        if assign_token and '=' in assign_token[0].string:
-            if self.linter.is_message_enabled('trailing-comma-tuple'):
-                self.add_message('trailing-comma-tuple',
-                                 line=token.start[0])
+            elif six.PY3 and is_trailing_comma(tokens, index):
+                if self.linter.is_message_enabled('trailing-comma-tuple'):
+                    self.add_message('trailing-comma-tuple',
+                                     line=token.start[0])
     def leave_module(self, _):
         self._init()
@@ -834,6 +816,46 @@ class LenChecker(checkers.BaseChecker):
                     self.add_message('len-as-condition', node=node)
+def is_trailing_comma(tokens, index):
+    """Check if the given token is a trailing comma
+
+    :param tokens: Sequence of modules tokens
+    :type tokens: list[tokenize.TokenInfo]
+    :param int index: Index of token under check in tokens
+    :returns: True if the token is a comma which trails an expression
+    :rtype: bool
+    """
+    token = tokens[index]
+    if token.exact_type != tokenize.COMMA:
+        return False
+    # Must have remaining tokens on the same line such as NEWLINE
+    left_tokens = itertools.islice(tokens, index + 1, None)
+    same_line_remaining_tokens = list(itertools.takewhile(
+        lambda other_token, _token=token: other_token.start[0] == _token.start[0],
+        left_tokens
+    ))
+    # Note: If the newline is tokenize.NEWLINE and not tokenize.NL
+    # then the newline denotes the end of expression
+    is_last_element = all(
+        other_token.type in (tokenize.NEWLINE, tokenize.COMMENT)
+        for other_token in same_line_remaining_tokens
+    )
+    if not same_line_remaining_tokens or not is_last_element:
+        return False
+    def get_curline_index_start():
+        """Get the index denoting the start of the current line"""
+        for subindex, token in enumerate(reversed(tokens[:index])):
+            # See Lib/tokenize.py and Lib/token.py in cpython for more info
+            if token.type in (tokenize.NEWLINE, tokenize.NL):
+                return index - subindex
+        return 0
+    curline_start = get_curline_index_start()
+    for prevtoken in tokens[curline_start:index]:
+        if '=' in prevtoken.string:
+            return True
+    return False
+
+
 def register(linter):
     """Required method to auto register this checker."""
     linter.register_checker(RefactoringChecker(linter))

pylint/checkers/stdlib.py

@@ -23,6 +23,8 @@ from pylint.checkers import utils
 OPEN_FILES = {'open', 'file'}
 UNITTEST_CASE = 'unittest.case'
 THREADING_THREAD = 'threading.Thread'
+COPY_COPY = 'copy.copy'
+OS_ENVIRON = 'os._Environ'
 if sys.version_info >= (3, 0):
     OPEN_MODULE = '_io'
 else:
@@ -105,6 +107,12 @@ class StdlibChecker(BaseChecker):
                   'The warning is emitted when a threading.Thread class '
                   'is instantiated without the target function being passed. '
                   'By default, the first parameter is the group param, not the target param. '),
+        'W1507': ('Using copy.copy(os.environ). Use os.environ.copy() '
+                  'instead. ',
+                  'shallow-copy-environ',
+                  'os.environ is not a dict object but proxy object, so '
+                  'shallow copy has still effects on original object. '
+                  'See https://bugs.python.org/issue15373 for reference. '),
     }
     deprecated = {
@@ -186,9 +194,17 @@ class StdlibChecker(BaseChecker):
         if not node.kwargs and node.args:
             self.add_message('bad-thread-instantiation', node=node)
+    def _check_shallow_copy_environ(self, node):
+        arg = utils.get_argument_from_call(node, position=0)
+        for inferred in arg.inferred():
+            if inferred.qname() == OS_ENVIRON:
+                self.add_message('shallow-copy-environ', node=node)
+                break
+
     @utils.check_messages('bad-open-mode', 'redundant-unittest-assert',
                           'deprecated-method',
-                          'bad-thread-instantiation')
+                          'bad-thread-instantiation',
+                          'shallow-copy-environ')
     def visit_call(self, node):
         """Visit a Call node."""
         try:
@@ -202,6 +218,8 @@ class StdlibChecker(BaseChecker):
                     self._check_redundant_assert(node, inferred)
                 if isinstance(inferred, astroid.ClassDef) and inferred.qname() == THREADING_THREAD:
                     self._check_bad_thread_instantiation(node)
+                if isinstance(inferred, astroid.FunctionDef) and inferred.qname() == COPY_COPY:
+                    self._check_shallow_copy_environ(node)
                 self._check_deprecated_method(node, inferred)
         except astroid.InferenceError:
             return

pylint/checkers/utils.py

@@ -878,3 +878,26 @@ def is_registered_in_singledispatch_function(node):
             return decorated_with(func_def, singledispatch_qnames)
     return False
+
+
+def get_node_last_lineno(node):
+    """
+    Get the last lineno of the given node. For a simple statement this will just be node.lineno,
+    but for a node that has child statements (e.g. a method) this will be the lineno of the last
+    child statement recursively.
+    """
+    # 'finalbody' is always the last clause in a try statement, if present
+    if getattr(node, 'finalbody', False):
+        return get_node_last_lineno(node.finalbody[-1])
+    # For if, while, and for statements 'orelse' is always the last clause.
+    # For try statements 'orelse' is the last in the absence of a 'finalbody'
+    if getattr(node, 'orelse', False):
+        return get_node_last_lineno(node.orelse[-1])
+    # try statements have the 'handlers' last if there is no 'orelse' or 'finalbody'
+    if getattr(node, 'handlers', False):
+        return get_node_last_lineno(node.handlers[-1])
+    # All compound statements have a 'body'
+    if getattr(node, 'body', False):
+        return get_node_last_lineno(node.body[-1])
+    # Not a compound statement
+    return node.lineno

pylint/extensions/_check_docs_utils.py

@@ -206,6 +206,11 @@ class Docstring(object):
 class SphinxDocstring(Docstring):
     re_type = r"[\w\.]+"
+    re_simple_container_type = r"""
+        {type}                        # a container type
+        [\(\[] [^\n\s]+ [\)\]]        # with the contents of the container
+    """.format(type=re_type)
+
     re_xref = r"""
         (?::\w+:)?                    # optional tag
         `{0}`                         # what to reference
@@ -221,14 +226,14 @@ class SphinxDocstring(Docstring):
         \s+                     # whitespace
         (?:                     # optional type declaration
-        ({type})
+        ({type}|{container_type})
         \s+
         )?
         (\w+)                   # Parameter name
         \s*                     # whitespace
         :                       # final colon
-        """.format(type=re_type)
+        """.format(type=re_type, container_type=re_simple_container_type)
     re_param_in_docstring = re.compile(re_param_raw, re.X | re.S)
     re_type_raw = r"""

pylint/lint.py

@@ -595,6 +595,10 @@ class PyLinter(config.OptionsManagerMixIn,
             for msg_id in self._checker_messages('python3'):
                 if msg_id.startswith('E'):
                     self.enable(msg_id)
+            config_parser = self.cfgfile_parser
+            if config_parser.has_option('MESSAGES CONTROL', 'disable'):
+                value = config_parser.get('MESSAGES CONTROL', 'disable')
+                self.global_set_option('disable', value)
         else:
             self.disable('python3')
         self.set_option('reports', False)
@@ -614,6 +618,10 @@ class PyLinter(config.OptionsManagerMixIn,
                     self.enable(msg_id)
                 else:
                     self.disable(msg_id)
+        config_parser = self.cfgfile_parser
+        if config_parser.has_option('MESSAGES CONTROL', 'disable'):
+            value = config_parser.get('MESSAGES CONTROL', 'disable')
+            self.global_set_option('disable', value)
         self._python3_porting_mode = True
     # block level option handling #############################################

pylint/test/extensions/test_check_docs.py

@@ -24,6 +24,9 @@ from pylint.extensions.docparams import DocstringParameterChecker
 class TestParamDocChecker(CheckerTestCase):
     """Tests for pylint_plugin.ParamDocChecker"""
     CHECKER_CLASS = DocstringParameterChecker
+    CONFIG = {
+        'accept_no_param_doc': False,
+    }
     def test_missing_func_params_in_sphinx_docstring(self):
         """Example of a function with missing Sphinx parameter documentation in
@@ -167,6 +170,7 @@ class TestParamDocChecker(CheckerTestCase):
         ):
             self.checker.visit_functiondef(node)
+    @set_config(accept_no_param_doc=True)
     def test_tolerate_no_param_documentation_at_all(self):
         """Example of a function with no parameter documentation at all
@@ -182,7 +186,6 @@ class TestParamDocChecker(CheckerTestCase):
         with self.assertNoMessages():
             self.checker.visit_functiondef(node)
-    @set_config(accept_no_param_doc=False)
     def test_don_t_tolerate_no_param_documentation_at_all(self):
         """Example of a function with no parameter documentation at all
@@ -207,7 +210,6 @@ class TestParamDocChecker(CheckerTestCase):
         ):
             self.checker.visit_functiondef(node)
-    @set_config(accept_no_param_doc=False)
     def test_see_tolerate_no_param_documentation_at_all(self):
         """Example for the usage of "For the parameters, see"
         to suppress missing-param warnings.
@@ -841,7 +843,6 @@ class TestParamDocChecker(CheckerTestCase):
         ):
             self._visit_methods_of_class(node)
-    @set_config(accept_no_param_doc=False)
     def test_see_sentence_for_constr_params_in_class(self):
         """Example usage of "For the parameters, see" in class docstring"""
         node = astroid.extract_node("""
@@ -859,7 +860,6 @@ class TestParamDocChecker(CheckerTestCase):
         with self.assertNoMessages():
             self._visit_methods_of_class(node)
-    @set_config(accept_no_param_doc=False)
     def test_see_sentence_for_constr_params_in_init(self):
         """Example usage of "For the parameters, see" in init docstring"""
         node = astroid.extract_node("""
@@ -1349,11 +1349,17 @@ class TestParamDocChecker(CheckerTestCase):
         with self.assertNoMessages():
             self.checker.visit_functiondef(node)
-    COMPLEX_TYPES = [
-        'int or str',
+    CONTAINER_TYPES = [
+        'dict(str,str)',
+        'dict[str,str]',
+        'tuple(int)',
+        'list[tokenize.TokenInfo]',
+    ]
+
+    COMPLEX_TYPES = CONTAINER_TYPES + [
         'dict(str, str)',
         'dict[str, str]',
-        'tuple(int)',
+        'int or str',
         'tuple(int or str)',
         'tuple(int) or list(int)',
         'tuple(int or str) or list(int or str)',
@@ -1414,6 +1420,22 @@ class TestParamDocChecker(CheckerTestCase):
         with self.assertNoMessages():
             self.checker.visit_functiondef(node)
+    @pytest.mark.parametrize('container_type', CONTAINER_TYPES)
+    def test_finds_compact_container_types_sphinx(self, container_type):
+        node = astroid.extract_node('''
+        def my_func(named_arg):
+            """The docstring
+
+            :param {0} named_arg: Returned
+
+            :returns: named_arg
+            :rtype: {0}
+            """
+            return named_arg
+        '''.format(container_type))
+        with self.assertNoMessages():
+            self.checker.visit_functiondef(node)
+
     def test_ignores_optional_specifier_numpy(self):
         node = astroid.extract_node('''
         def do_something(param, param2='all'):

pylint/test/functional/trailing_comma_tuple.py

@@ -3,6 +3,7 @@
 AAA = 1, # [trailing-comma-tuple]
 BBB = "aaaa", # [trailing-comma-tuple]
 CCC="aaa", # [trailing-comma-tuple]
+FFF=['f'], # [trailing-comma-tuple]
 BBB = 1, 2
 CCC = (1, 2, 3)

pylint/test/functional/trailing_comma_tuple.txt

 trailing-comma-tuple:3::Disallow trailing comma tuple
 trailing-comma-tuple:4::Disallow trailing comma tuple
 trailing-comma-tuple:5::Disallow trailing comma tuple
+trailing-comma-tuple:6::Disallow trailing comma tuple

pylint/test/functional/wrong_import_order.py

 """Checks import order rule"""
 # pylint: disable=unused-import,relative-import,ungrouped-imports,import-error,no-name-in-module,relative-beyond-top-level
+from __future__ import absolute_import
 try:
     from six.moves import configparser
 except ImportError:

pylint/test/functional/wrong_import_order.txt

-wrong-import-order:11::standard import "import os.path" should be placed before "import six"
-wrong-import-order:13::standard import "import sys" should be placed before "import six"
-wrong-import-order:14::standard import "import datetime" should be placed before "import six"
-wrong-import-order:17::external import "import totally_missing" should be placed before "from .package import Class"
-wrong-import-order:19::external import "import astroid" should be placed before "from .package import Class"
+wrong-import-order:12::standard import "import os.path" should be placed before "import six"
+wrong-import-order:14::standard import "import sys" should be placed before "import six"
+wrong-import-order:15::standard import "import datetime" should be placed before "import six"
+wrong-import-order:18::first party import "import totally_missing" should be placed before "from .package import Class"
+wrong-import-order:20::third party import "import astroid" should be placed before "import unused_import"

pylint/test/functional/wrong_import_order2.py

@@ -7,7 +7,7 @@ import os
 from sys import argv
 # external imports
-import lxml
+import isort
 from six import moves

pylint/test/regrtest_data/py3k-disabled.rc

+[MESSAGES CONTROL]
+disable=no-absolute-import

pylint/test/regrtest_data/py3k_errors_and_warnings.py

+"""Contains both normal error messages and Python3 porting error messages."""
+# pylint: disable=too-few-public-methods
+
+# error: import missing `from __future__ import absolute_import`
+import sys
+
+# error: Use raise ErrorClass(args) instead of raise ErrorClass, args.
+raise Exception, 1
+
+class Test(object):
+    """dummy"""
+
+    def __init__(self):
+        # warning: Calling a dict.iter*() method
+        {1: 2}.iteritems()
+        return 42
+
+# error: print statement used
+print 'not in python3'