0.2.4-0.1 (patches applied)applied/0.2.4-0.1 applied/ubuntu/bionic-proposed applied/ubuntu/bionic

Imported using git-ubuntu import.
author: Matthias Klose <doko@debian.org> 2018-02-06 17:27:17 +0100
committer: git-ubuntu importer <ubuntu-devel-discuss@lists.ubuntu.com> 2018-02-06 18:10:11 +0000
commit: 2232060eacf87c0833316739e97e765df03e7792 (patch)
tree: 3c8075a724e6a53f6aac75f8673de16d9ba88e66
parent: da0cdcafe1c50c66fa73fecd52f6cab9faa76a5c (diff)
parent: 0abadd07d7b733e1cc9a9079265df5e95a57a461 (diff)
29 files changed, 388 insertions, 72 deletions
diff --git a/AUTHORS b/AUTHORS
index b8dd119..e5ddd32 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -11,31 +11,38 @@ Alphabetical list of contributors:
 * Alexander Beedie <ayembee@gmail.com>
 * Alexey Malyshev <nostrict@gmail.com>
 * Andrew Tipton <andrew.tipton@compareglobalgroup.com>
+* atronah <atronah.ds@gmail.com>
 * casey <casey@cloudera.com>
 * Cauê Beloni <cbeloni@gmail.com>
 * Cristian Orellana <cristiano@groupon.com>
 * Dag Wieers <dag@wieers.com>
 * Darik Gamble <darik.gamble@gmail.com>
+* Demetrio92 <Demetrio.Rodriguez.T@gmail.com>
 * Dennis Taylor <dennis.taylor@clio.com>
 * Florian Bauer <florian.bauer@zmdi.com>
 * Gavin Wahl <gwahl@fusionbox.com>
 * JacekPliszka <Jacek.Pliszka@gmail.com>
 * Jesús Leganés Combarro "Piranna" <piranna@gmail.com>
+* Jon Dufresne <jon.dufresne@gmail.com>
 * Kevin Jing Qiu <kevin.jing.qiu@gmail.com>
 * koljonen <koljonen@outlook.com>
 * Michael Schuller <chick@mschuller.net>
 * Mike Amy <cocoade@googlemail.com>
 * mulos <daniel.strackbein@gmail.com>
+* Oleg Broytman <phd@phdru.name>
 * Piet Delport <pjdelport@gmail.com>
 * Prudhvi Vatala <pvatala@gmail.com>
 * quest <quest@wonky.windwards.net>
 * Robert Nix <com.github@rnix.org>
 * Rocky Meza <rmeza@fusionbox.com>
+* Romain Rigaux <romain.rigaux@gmail.com>
+* Rowan Seymour <rowanseymour@gmail.com>
 * Ryan Wooden <rygwdn@gmail.com>
 * saaj <id@saaj.me>
 * Shen Longxing <shenlongxing2012@gmail.com>
 * Sjoerd Job Postmus
 * spigwitmer <itgpmc@gmail.com>
+* Tao Wang <twang2218@gmail.com>
 * Tenghuan <tenghuanhe@gmail.com>
 * Tim Graham <timograham@gmail.com>
 * Victor Hahn <info@victor-hahn.de>
diff --git a/CHANGELOG b/CHANGELOG
index 88e3a8b..18bb9c1 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,40 @@
+Release 0.2.4 (Sep 27, 2017)
+----------------------------
+
+Enhancements
+
+* Add more keywords for MySQL table options (pr328, pr333, by phdru).
+* Add more PL/pgSQL keywords (pr357, by Demetrio92).
+* Improve parsing of floats (pr330, by atronah).
+
+Bug Fixes
+
+* Fix parsing of MySQL table names starting with digits (issue337).
+* Fix detection of identifiers using comparisons (issue327).
+* Fix parsing of UNION ALL after WHERE (issue349).
+* Fix handling of semicolon in assignments (issue359, issue358).
+
+
+
+Release 0.2.3 (Mar 02, 2017)
+----------------------------
+
+Enhancements
+
+* New command line option "--encoding" (by twang2218, pr317).
+* Support CONCURRENTLY keyword (issue322, by rowanseymour).
+
+Bug Fixes
+
+* Fix some edge-cases when parsing invalid SQL statements.
+* Fix indentation of LIMIT (by romainr, issue320).
+* Fix parsing of INTO keyword (issue324).
+
+Internal Changes
+
+* Several improvements regarding encodings.
+
+
 Release 0.2.2 (Oct 22, 2016)
 ----------------------------
 
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..de414c5
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,25 @@
+Copyright (c) 2016, Andi Albrecht <albrecht.andi@gmail.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the authors nor the names of its contributors may be
+      used to endorse or promote products derived from this software without
+      specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/MANIFEST.in b/MANIFEST.in
index 585ab3b..8043b35 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,10 +2,10 @@ recursive-include docs source/*
 include docs/sqlformat.1
 include docs/Makefile
 recursive-include tests *.py *.sql
-include COPYING
+include LICENSE
 include TODO
 include AUTHORS
 include CHANGELOG
 include Makefile
-include pytest.ini
+include setup.cfg
 include tox.ini
diff --git a/PKG-INFO b/PKG-INFO
index 79637ee..52063a4 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: sqlparse
-Version: 0.2.2
+Version: 0.2.4
 Summary: Non-validating SQL parser
 Home-page: https://github.com/andialbrecht/sqlparse
 Author: Andi Albrecht
@@ -68,5 +68,6 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.3
 Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
 Classifier: Topic :: Database
 Classifier: Topic :: Software Development
diff --git a/README.rst b/README.rst
index f397c01..06b9e03 100644
--- a/README.rst
+++ b/README.rst
@@ -10,7 +10,15 @@ sqlparse is a non-validating SQL parser module for Python.
 Install
 -------
 
-Run::
+From pip, run::
+
+    $ pip install --upgrade sqlparse
+
+Consider using the ``--user`` option_.
+
+.. _option: https://pip.pypa.io/en/latest/user_guide/#user-installs
+
+From the repository, run::
 
   python setup.py install
 
diff --git a/debian/changelog b/debian/changelog
index d5bf509..6af0693 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,15 @@
+sqlparse (0.2.4-0.1) unstable; urgency=medium
+
+  * Non-maintainer upload.
+  * New upstream version.
+  * Build-depend on 2to3. Closes: #884670.
+  * Add 2to3 to autopkg test depends. Closes: #889007.
+  * Bump standards version.
+  * Mark python-sqlparse-doc as M-A: foreign.
+  * Build-depend on python3-sphinx instead of python-sphinx.
+
+ -- Matthias Klose <doko@debian.org>  Tue, 06 Feb 2018 17:27:17 +0100
+
 sqlparse (0.2.2-1) unstable; urgency=medium
 
   [ Ondřej Nový ]
diff --git a/debian/control b/debian/control
index 47af5bd..bbd0769 100644
--- a/debian/control
+++ b/debian/control
@@ -5,11 +5,11 @@ Maintainer: Andriy Senkovych <jolly_roger@itblog.org.ua>
 Uploaders: Debian Python Modules Team <python-modules-team@lists.alioth.debian.org>
 Build-Depends: debhelper (>= 9), dh-python,
  python-all (>= 2.6.5-13~), python3-all (>= 3.2),
- python-sphinx (>= 1.0.7+dfsg),
+ python3-sphinx (>= 1.0.7+dfsg),
  python-pytest, python3-pytest,
  python-setuptools, python3-setuptools, pypy-setuptools,
- pypy
-Standards-Version: 3.9.8
+ pypy, 2to3
+Standards-Version: 4.1.3
 Homepage: https://github.com/andialbrecht/sqlparse
 Vcs-Git: https://anonscm.debian.org/git/python-modules/packages/sqlparse.git
 Vcs-Browser: https://anonscm.debian.org/cgit/python-modules/packages/sqlparse.git
@@ -49,6 +49,7 @@ Description: non-validating SQL parser for PyPy
 
 Package: python-sqlparse-doc
 Architecture: all
+Multi-Arch: foreign
 Section: doc
 Replaces: python-sqlparse(<< 0.1.6-1)
 Breaks: python-sqlparse (<< 0.1.6-1)
diff --git a/debian/tests/control b/debian/tests/control
index eee3c21..b5da0f2 100644
--- a/debian/tests/control
+++ b/debian/tests/control
@@ -1,6 +1,6 @@
 Tests: python-sqlparse
-Depends: python-all, python-sqlparse, python-pytest
+Depends: python-all, python-sqlparse, python-pytest, 2to3
 
 Tests: python3-sqlparse
-Depends: python-all, python3-sqlparse, python3-pytest
+Depends: python-all, python3-sqlparse, python3-pytest, 2to3
 Restrictions: allow-stderr
diff --git a/setup.cfg b/setup.cfg
index cb71d1c..1920fbf 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,7 +1,7 @@
-[wheel]
+[bdist_wheel]
 universal = 1
 
-[pytest]
+[tool:pytest]
 xfail_strict = True
 
 [flake8]
@@ -19,5 +19,4 @@ omit =
 [egg_info]
 tag_build = 
 tag_date = 0
-tag_svn_revision = 0
 
diff --git a/setup.py b/setup.py
index 3033305..c40ee71 100644
--- a/setup.py
+++ b/setup.py
@@ -95,6 +95,7 @@ setup(
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Topic :: Database',
         'Topic :: Software Development',
     ],
diff --git a/sqlparse.egg-info/PKG-INFO b/sqlparse.egg-info/PKG-INFO
index 79637ee..52063a4 100644
--- a/sqlparse.egg-info/PKG-INFO
+++ b/sqlparse.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: sqlparse
-Version: 0.2.2
+Version: 0.2.4
 Summary: Non-validating SQL parser
 Home-page: https://github.com/andialbrecht/sqlparse
 Author: Andi Albrecht
@@ -68,5 +68,6 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.3
 Classifier: Programming Language :: Python :: 3.4
 Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
 Classifier: Topic :: Database
 Classifier: Topic :: Software Development
diff --git a/sqlparse.egg-info/SOURCES.txt b/sqlparse.egg-info/SOURCES.txt
index e635c2f..a80bdaa 100644
--- a/sqlparse.egg-info/SOURCES.txt
+++ b/sqlparse.egg-info/SOURCES.txt
@@ -1,5 +1,6 @@
 AUTHORS
 CHANGELOG
+LICENSE
 MANIFEST.in
 Makefile
 README.rst
@@ -49,6 +50,7 @@ tests/conftest.py
 tests/test_cli.py
 tests/test_format.py
 tests/test_grouping.py
+tests/test_keywords.py
 tests/test_parse.py
 tests/test_regressions.py
 tests/test_split.py
@@ -57,6 +59,8 @@ tests/files/_Make_DirEntry.sql
 tests/files/begintag.sql
 tests/files/begintag_2.sql
 tests/files/dashcomment.sql
+tests/files/encoding_gbk.sql
+tests/files/encoding_utf8.sql
 tests/files/function.sql
 tests/files/function_psql.sql
 tests/files/function_psql2.sql
diff --git a/sqlparse/__init__.py b/sqlparse/__init__.py
index b8dbf48..237e303 100644
--- a/sqlparse/__init__.py
+++ b/sqlparse/__init__.py
@@ -17,7 +17,7 @@ from sqlparse import formatter
 
 from sqlparse.compat import text_type
 
-__version__ = '0.2.2'
+__version__ = '0.2.4'
 __all__ = ['engine', 'filters', 'formatter', 'sql', 'tokens', 'cli']
 
 
diff --git a/sqlparse/cli.py b/sqlparse/cli.py
index bd2e1b8..ad6bc7a 100755
--- a/sqlparse/cli.py
+++ b/sqlparse/cli.py
@@ -21,6 +21,8 @@ Why does this file exist, and why not put this in __main__?
 
 import argparse
 import sys
+from io import TextIOWrapper
+from codecs import open, getreader
 
 import sqlparse
 from sqlparse.compat import PY2
@@ -125,6 +127,12 @@ def create_parser():
         type=bool,
         help='Insert linebreak before comma (default False)')
 
+    group.add_argument(
+        '--encoding',
+        dest='encoding',
+        default='utf-8',
+        help='Specify the input encoding (default utf-8)')
+
     return parser
 
 
@@ -139,18 +147,24 @@ def main(args=None):
     args = parser.parse_args(args)
 
     if args.filename == '-':  # read from stdin
-        data = sys.stdin.read()
+        if PY2:
+            data = getreader(args.encoding)(sys.stdin).read()
+        else:
+            data = TextIOWrapper(
+                sys.stdin.buffer, encoding=args.encoding).read()
     else:
         try:
-            # TODO: Needs to deal with encoding
-            data = ''.join(open(args.filename).readlines())
+            with open(args.filename, 'r', args.encoding) as f:
+                data = ''.join(f.readlines())
         except IOError as e:
             return _error(
                 u'Failed to read {0}: {1}'.format(args.filename, e))
 
+    close_stream = False
     if args.outfile:
         try:
-            stream = open(args.outfile, 'w')
+            stream = open(args.outfile, 'w', args.encoding)
+            close_stream = True
         except IOError as e:
             return _error(u'Failed to open {0}: {1}'.format(args.outfile, e))
     else:
@@ -163,8 +177,8 @@ def main(args=None):
         return _error(u'Invalid options: {0}'.format(e))
 
     s = sqlparse.format(data, **formatter_opts)
-    if PY2:
-        s = s.encode('utf-8', 'replace')
     stream.write(s)
     stream.flush()
+    if close_stream:
+        stream.close()
     return 0
diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py
index 5fa3909..fa87c9f 100644
--- a/sqlparse/engine/grouping.py
+++ b/sqlparse/engine/grouping.py
@@ -134,7 +134,7 @@ def group_assignment(tlist):
         return token.match(T.Assignment, ':=')
 
     def valid(token):
-        return token is not None
+        return token is not None and token.ttype not in (T.Keyword)
 
     def post(tlist, pidx, tidx, nidx):
         m_semicolon = T.Punctuation, ';'
@@ -274,7 +274,7 @@ def group_where(tlist):
 @recurse()
 def group_aliased(tlist):
     I_ALIAS = (sql.Parenthesis, sql.Function, sql.Case, sql.Identifier,
-               sql.Operation)
+               sql.Operation, sql.Comparison)
 
     tidx, token = tlist.token_next_by(i=I_ALIAS, t=T.Number)
     while token:
@@ -346,10 +346,10 @@ def group(stmt):
         group_order,
         group_typecasts,
         group_operator,
+        group_comparison,
         group_as,
         group_aliased,
         group_assignment,
-        group_comparison,
 
         align_comments,
         group_identifier_list,
@@ -380,7 +380,7 @@ def _group(tlist, cls, match,
 
         if match(token):
             nidx, next_ = tlist.token_next(tidx)
-            if valid_prev(prev_) and valid_next(next_):
+            if prev_ and valid_prev(prev_) and valid_next(next_):
                 from_idx, to_idx = post(tlist, pidx, tidx, nidx)
                 grp = tlist.group_tokens(cls, from_idx, to_idx, extend=extend)
 
diff --git a/sqlparse/filters/reindent.py b/sqlparse/filters/reindent.py
index ff1b211..f077849 100644
--- a/sqlparse/filters/reindent.py
+++ b/sqlparse/filters/reindent.py
@@ -30,7 +30,7 @@ class ReindentFilter(object):
 
         for t in self._curr_stmt.flatten():
             if t == token:
-                raise StopIteration
+                break
             yield t
 
     @property
@@ -51,7 +51,7 @@ class ReindentFilter(object):
     def _next_token(self, tlist, idx=-1):
         split_words = ('FROM', 'STRAIGHT_JOIN$', 'JOIN$', 'AND', 'OR',
                        'GROUP', 'ORDER', 'UNION', 'VALUES',
-                       'SET', 'BETWEEN', 'EXCEPT', 'HAVING')
+                       'SET', 'BETWEEN', 'EXCEPT', 'HAVING', 'LIMIT')
         m_split = T.Keyword, split_words, True
         tidx, token = tlist.token_next_by(m=m_split, idx=idx)
 
@@ -162,12 +162,14 @@ class ReindentFilter(object):
                 with offset(self, len("WHEN ")):
                     self._process_default(tlist)
             end_idx, end = tlist.token_next_by(m=sql.Case.M_CLOSE)
-            tlist.insert_before(end_idx, self.nl())
+            if end_idx is not None:
+                tlist.insert_before(end_idx, self.nl())
 
     def _process_default(self, tlist, stmts=True):
         self._split_statements(tlist) if stmts else None
         self._split_kwds(tlist)
-        [self._process(sgroup) for sgroup in tlist.get_sublists()]
+        for sgroup in tlist.get_sublists():
+            self._process(sgroup)
 
     def process(self, stmt):
         self._curr_stmt = stmt
diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py
index 48d37f0..a1242ab 100644
--- a/sqlparse/keywords.py
+++ b/sqlparse/keywords.py
@@ -14,6 +14,7 @@ def is_keyword(value):
     val = value.upper()
     return (KEYWORDS_COMMON.get(val) or
             KEYWORDS_ORACLE.get(val) or
+            KEYWORDS_PLPGSQL.get(val) or
             KEYWORDS.get(val, tokens.Name)), value
 
 
@@ -35,7 +36,7 @@ SQL_REGEX = {
 
         (r"`(``|[^`])*`", tokens.Name),
         (r"´(´´|[^´])*´", tokens.Name),
-        (r'(\$(?:[_A-Z]\w*)?\$)[\s\S]*?\1', tokens.Literal),
+        (r'(\$(?:[_A-ZÀ-Ü]\w*)?\$)[\s\S]*?\1', tokens.Literal),
 
         (r'\?', tokens.Name.Placeholder),
         (r'%(\(\w+\))?s', tokens.Name.Placeholder),
@@ -47,21 +48,20 @@ SQL_REGEX = {
         # is never a functino, see issue183
         (r'(CASE|IN|VALUES|USING)\b', tokens.Keyword),
 
-        (r'(@|##|#)[A-Z]\w+', tokens.Name),
+        (r'(@|##|#)[A-ZÀ-Ü]\w+', tokens.Name),
 
         # see issue #39
         # Spaces around period `schema . name` are valid identifier
         # TODO: Spaces before period not implemented
-        (r'[A-Z]\w*(?=\s*\.)', tokens.Name),  # 'Name'   .
-        (r'(?<=\.)[A-Z]\w*', tokens.Name),  # .'Name'
-        (r'[A-Z]\w*(?=\()', tokens.Name),  # side effect: change kw to func
-
-        # TODO: `1.` and `.1` are valid numbers
+        (r'[A-ZÀ-Ü]\w*(?=\s*\.)', tokens.Name),  # 'Name'   .
+        # FIXME(atronah): never match,
+        # because `re.match` doesn't work with lookbehind regexp feature
+        (r'(?<=\.)[A-ZÀ-Ü]\w*', tokens.Name),  # .'Name'
+        (r'[A-ZÀ-Ü]\w*(?=\()', tokens.Name),  # side effect: change kw to func
         (r'-?0x[\dA-F]+', tokens.Number.Hexadecimal),
         (r'-?\d*(\.\d+)?E-?\d+', tokens.Number.Float),
-        (r'-?\d*\.\d+', tokens.Number.Float),
-        (r'-?\d+', tokens.Number.Integer),
-
+        (r'-?(\d+(\.\d*)|\.\d+)', tokens.Number.Float),
+        (r'-?\d+(?![_A-ZÀ-Ü])', tokens.Number.Integer),
         (r"'(''|\\\\|\\'|[^'])*'", tokens.String.Single),
         # not a real string literal in ANSI SQL:
         (r'(""|".*?[^\\]")', tokens.String.Symbol),
@@ -77,7 +77,7 @@ SQL_REGEX = {
         (r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL),
         (r'DOUBLE\s+PRECISION\b', tokens.Name.Builtin),
 
-        (r'[_A-Z][_$#\w]*', is_keyword),
+        (r'[0-9_A-ZÀ-Ü][_$#\w]*', is_keyword),
 
         (r'[;:()\[\],\.]', tokens.Punctuation),
         (r'[<>=~!]+', tokens.Operator.Comparison),
@@ -114,6 +114,7 @@ KEYWORDS = {
     'ATOMIC': tokens.Keyword,
     'AUDIT': tokens.Keyword,
     'AUTHORIZATION': tokens.Keyword,
+    'AUTO_INCREMENT': tokens.Keyword,
     'AVG': tokens.Keyword,
 
     'BACKWARD': tokens.Keyword,
@@ -142,6 +143,7 @@ KEYWORDS = {
     'CHARACTER_SET_NAME': tokens.Keyword,
     'CHARACTER_SET_SCHEMA': tokens.Keyword,
     'CHAR_LENGTH': tokens.Keyword,
+    'CHARSET': tokens.Keyword,
     'CHECK': tokens.Keyword,
     'CHECKED': tokens.Keyword,
     'CHECKPOINT': tokens.Keyword,
@@ -167,6 +169,7 @@ KEYWORDS = {
     'COMMIT': tokens.Keyword.DML,
     'COMMITTED': tokens.Keyword,
     'COMPLETION': tokens.Keyword,
+    'CONCURRENTLY': tokens.Keyword,
     'CONDITION_NUMBER': tokens.Keyword,
     'CONNECT': tokens.Keyword,
     'CONNECTION': tokens.Keyword,
@@ -237,6 +240,7 @@ KEYWORDS = {
     'ENCODING': tokens.Keyword,
     'ENCRYPTED': tokens.Keyword,
     'END-EXEC': tokens.Keyword,
+    'ENGINE': tokens.Keyword,
     'EQUALS': tokens.Keyword,
     'ESCAPE': tokens.Keyword,
     'EVERY': tokens.Keyword,
@@ -513,7 +517,7 @@ KEYWORDS = {
     'SQLWARNING': tokens.Keyword,
     'STABLE': tokens.Keyword,
     'START': tokens.Keyword.DML,
-    'STATE': tokens.Keyword,
+    # 'STATE': tokens.Keyword,
     'STATEMENT': tokens.Keyword,
     'STATIC': tokens.Keyword,
     'STATISTICS': tokens.Keyword,
@@ -635,7 +639,7 @@ KEYWORDS = {
     'SERIAL8': tokens.Name.Builtin,
     'SIGNED': tokens.Name.Builtin,
     'SMALLINT': tokens.Name.Builtin,
-    'SYSDATE': tokens.Name.Builtin,
+    'SYSDATE': tokens.Name,
     'TEXT': tokens.Name.Builtin,
     'TINYINT': tokens.Name.Builtin,
     'UNSIGNED': tokens.Name.Builtin,
@@ -721,7 +725,8 @@ KEYWORDS_ORACLE = {
     'FREELIST': tokens.Keyword,
     'FREELISTS': tokens.Keyword,
 
-    'GROUPS': tokens.Keyword,
+    # groups seems too common as table name
+    # 'GROUPS': tokens.Keyword,
 
     'INDICATOR': tokens.Keyword,
     'INITRANS': tokens.Keyword,
@@ -793,4 +798,20 @@ KEYWORDS_ORACLE = {
     'TRIGGERS': tokens.Keyword,
 
     'UNLIMITED': tokens.Keyword,
+    'UNLOCK': tokens.Keyword,
+}
+
+# PostgreSQL Syntax
+KEYWORDS_PLPGSQL = {
+    'PARTITION': tokens.Keyword,
+    'OVER': tokens.Keyword,
+    'PERFORM': tokens.Keyword,
+    'NOTICE': tokens.Keyword,
+    'PLPGSQL': tokens.Keyword,
+    'INHERIT': tokens.Keyword,
+    'INDEXES': tokens.Keyword,
+
+    'FOR': tokens.Keyword,
+    'IN': tokens.Keyword,
+    'LOOP': tokens.Keyword,
 }
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index 914b520..60e43da 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -43,12 +43,13 @@ class Lexer(object):
         if isinstance(text, text_type):
             pass
         elif isinstance(text, bytes_type):
-            try:
-                text = text.decode()
-            except UnicodeDecodeError:
-                if not encoding:
-                    encoding = 'unicode-escape'
+            if encoding:
                 text = text.decode(encoding)
+            else:
+                try:
+                    text = text.decode('utf-8')
+                except UnicodeDecodeError:
+                    text = text.decode('unicode-escape')
         else:
             raise TypeError(u"Expected text or file-like object, got {!r}".
                             format(type(text)))
diff --git a/sqlparse/sql.py b/sqlparse/sql.py
index c0f8306..f698782 100644
--- a/sqlparse/sql.py
+++ b/sqlparse/sql.py
@@ -526,8 +526,9 @@ class Comment(TokenList):
 class Where(TokenList):
     """A WHERE clause."""
     M_OPEN = T.Keyword, 'WHERE'
-    M_CLOSE = T.Keyword, ('ORDER', 'GROUP', 'LIMIT', 'UNION', 'EXCEPT',
-                          'HAVING', 'RETURNING')
+    M_CLOSE = T.Keyword, (
+        'ORDER', 'GROUP', 'LIMIT', 'UNION', 'UNION ALL', 'EXCEPT',
+        'HAVING', 'RETURNING', 'INTO')
 
 
 class Case(TokenList):
diff --git a/sqlparse/utils.py b/sqlparse/utils.py
index ee3e982..ffa029b 100644
--- a/sqlparse/utils.py
+++ b/sqlparse/utils.py
@@ -95,9 +95,9 @@ def imt(token, i=None, m=None, t=None):
         return False
     elif clss and isinstance(token, clss):
         return True
-    elif mpatterns and any((token.match(*pattern) for pattern in mpatterns)):
+    elif mpatterns and any(token.match(*pattern) for pattern in mpatterns):
         return True
-    elif types and any([token.ttype in ttype for ttype in types]):
+    elif types and any(token.ttype in ttype for ttype in types):
         return True
     else:
         return False
diff --git a/tests/files/encoding_gbk.sql b/tests/files/encoding_gbk.sql
new file mode 100644
index 0000000..a613229
--- /dev/null
+++ b/tests/files/encoding_gbk.sql
@@ -0,0 +1,3 @@
+select *
+from foo
+where bar = '������ϲ�����Լ���'
+\ No newline at end of file
diff --git a/tests/files/encoding_utf8.sql b/tests/files/encoding_utf8.sql
new file mode 100644
index 0000000..26e7ad4
--- /dev/null
+++ b/tests/files/encoding_utf8.sql
@@ -0,0 +1,3 @@
+select *
+from foo
+where bar = '齐天大圣.カラフルな雲.사랑해요'
+\ No newline at end of file
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 77a764e..18c6fcb 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -73,3 +73,72 @@ def test_script():
     # Call with the --help option as a basic sanity check.
     cmd = "{0:s} -m sqlparse.cli --help".format(sys.executable)
     assert subprocess.call(cmd.split()) == 0
+
+
+def test_encoding_utf8_stdout(filepath, load_file, capfd):
+    path = filepath('encoding_utf8.sql')
+    expected = load_file('encoding_utf8.sql', 'utf-8')
+    sys.stdout.encoding = 'utf-8'
+    sqlparse.cli.main([path])
+    out, _ = capfd.readouterr()
+    assert out == expected
+
+
+def test_encoding_utf8_output_file(filepath, load_file, tmpdir):
+    in_path = filepath('encoding_utf8.sql')
+    expected = load_file('encoding_utf8.sql', 'utf-8')
+    out_path = tmpdir.dirname + '/encoding_utf8.out.sql'
+    sqlparse.cli.main([in_path, '-o', out_path])
+    out = load_file(out_path, 'utf-8')
+    assert out == expected
+
+
+def test_encoding_gbk_stdout(filepath, load_file, capfd):
+    path = filepath('encoding_gbk.sql')
+    expected = load_file('encoding_gbk.sql', 'gbk')
+    sys.stdout.encoding = 'gbk'
+    sqlparse.cli.main([path, '--encoding', 'gbk'])
+    out, _ = capfd.readouterr()
+    assert out == expected
+
+
+def test_encoding_gbk_output_file(filepath, load_file, tmpdir):
+    in_path = filepath('encoding_gbk.sql')
+    expected = load_file('encoding_gbk.sql', 'gbk')
+    out_path = tmpdir.dirname + '/encoding_gbk.out.sql'
+    sqlparse.cli.main([in_path, '--encoding', 'gbk', '-o', out_path])
+    out = load_file(out_path, 'gbk')
+    assert out == expected
+
+
+def test_encoding_stdin_utf8(filepath, load_file, capfd):
+    path = filepath('encoding_utf8.sql')
+    expected = load_file('encoding_utf8.sql', 'utf-8')
+    old_stdin = sys.stdin
+    sys.stdin = open(path, 'r')
+    sys.stdout.encoding = 'utf-8'
+    sqlparse.cli.main(['-'])
+    sys.stdin = old_stdin
+    out, _ = capfd.readouterr()
+    assert out == expected
+
+
+def test_encoding_stdin_gbk(filepath, load_file, capfd):
+    path = filepath('encoding_gbk.sql')
+    expected = load_file('encoding_gbk.sql', 'gbk')
+    old_stdin = sys.stdin
+    with open(path, 'r') as stream:
+        sys.stdin = stream
+        sys.stdout.encoding = 'gbk'
+        sqlparse.cli.main(['-', '--encoding', 'gbk'])
+        sys.stdin = old_stdin
+    out, _ = capfd.readouterr()
+    assert out == expected
+
+
+def test_encoding(filepath, capsys):
+    path = filepath('test_cp1251.sql')
+    expected = u'insert into foo values (1); -- Песня про надежду\n'
+    sqlparse.cli.main([path, '--encoding=cp1251'])
+    out, _ = capsys.readouterr()
+    assert out == expected
diff --git a/tests/test_grouping.py b/tests/test_grouping.py
index 20151a1..1f3a19e 100644
--- a/tests/test_grouping.py
+++ b/tests/test_grouping.py
@@ -65,6 +65,20 @@ def test_grouping_identifiers():
 
 
 @pytest.mark.parametrize('s', [
+    '1 as f',
+    'foo as f',
+    'foo f',
+    '1/2 as f',
+    '1/2 f',
+    '1<2 as f',  # issue327
+    '1<2 f',
+])
+def test_simple_identifiers(s):
+    parsed = sqlparse.parse(s)[0]
+    assert isinstance(parsed.tokens[0], sql.Identifier)
+
+
+@pytest.mark.parametrize('s', [
     'foo, bar',
     'sum(a), sum(b)',
     'sum(a) as x, b as y',
@@ -202,6 +216,15 @@ def test_grouping_where():
     assert isinstance(p.tokens[-1].tokens[0].tokens[-2], sql.Where)
 
 
+@pytest.mark.parametrize('s', (
+    'select 1 where 1 = 2 union select 2',
+    'select 1 where 1 = 2 union all select 2',
+))
+def test_grouping_where_union(s):
+    p = sqlparse.parse(s)[0]
+    assert p.tokens[5].value.startswith('union')
+
+
 def test_returning_kw_ends_where_clause():
     s = 'delete from foo where x > y returning z'
     p = sqlparse.parse(s)[0]
@@ -210,6 +233,14 @@ def test_returning_kw_ends_where_clause():
     assert p.tokens[7].value == 'returning'
 
 
+def test_into_kw_ends_where_clause():  # issue324
+    s = 'select * from foo where a = 1 into baz'
+    p = sqlparse.parse(s)[0]
+    assert isinstance(p.tokens[8], sql.Where)
+    assert p.tokens[9].ttype == T.Keyword
+    assert p.tokens[9].value == 'into'
+
+
 @pytest.mark.parametrize('sql, expected', [
     # note: typecast needs to be 2nd token for this test
     ('select foo::integer from bar', 'integer'),
@@ -250,6 +281,16 @@ def test_grouping_alias_case():
     assert p.tokens[0].get_alias() == 'foo'
 
 
+def test_grouping_subquery_no_parens():
+    # Not totally sure if this is the right approach...
+    # When a THEN clause contains a subquery w/o parens around it *and*
+    # a WHERE condition, the WHERE grouper consumes END too.
+    # This takes makes sure that it doesn't fail.
+    p = sqlparse.parse('CASE WHEN 1 THEN select 2 where foo = 1 end')[0]
+    assert len(p.tokens) == 1
+    assert isinstance(p.tokens[0], sql.Case)
+
+
 def test_grouping_alias_returns_none():
     # see issue185
     p = sqlparse.parse('foo.bar')[0]
diff --git a/tests/test_keywords.py b/tests/test_keywords.py
new file mode 100644
index 0000000..c197f36
--- /dev/null
+++ b/tests/test_keywords.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+import pytest
+
+from sqlparse import tokens
+from sqlparse.keywords import SQL_REGEX
+
+
+class TestSQLREGEX:
+    @pytest.mark.parametrize('number', ['1.0', '-1.0',
+                                        '1.', '-1.',
+                                        '.1', '-.1'])
+    def test_float_numbers(self, number):
+        ttype = next(tt for action, tt in SQL_REGEX if action(number))
+        assert tokens.Number.Float == ttype
diff --git a/tests/test_parse.py b/tests/test_parse.py
index 0632889..4ba6ca1 100644
--- a/tests/test_parse.py
+++ b/tests/test_parse.py
@@ -135,11 +135,14 @@ def test_quoted_identifier():
     assert t[2].get_real_name() == 'y'
 
 
-@pytest.mark.parametrize('name', ['foo', '_foo'])
+@pytest.mark.parametrize('name', [
+    'foo', '_foo',  # issue175
+    '1_data',  # valid MySQL table name, see issue337
+])
 def test_valid_identifier_names(name):
-    # issue175
     t = sqlparse.parse(name)[0].tokens
     assert isinstance(t[0], sql.Identifier)
+    assert t[0].get_name() == name
 
 
 def test_psql_quotation_marks():
diff --git a/tests/test_regressions.py b/tests/test_regressions.py
index ba908d3..406328c 100644
--- a/tests/test_regressions.py
+++ b/tests/test_regressions.py
@@ -43,13 +43,14 @@ def test_issue34(value):
 
 
 def test_issue35():
-    # missing space before LIMIT
+    # missing space before LIMIT. Updated for #321
     sql = sqlparse.format("select * from foo where bar = 1 limit 1",
                           reindent=True)
     assert sql == "\n".join([
         "select *",
         "from foo",
-        "where bar = 1 limit 1"])
+        "where bar = 1",
+        "limit 1"])
 
 
 def test_issue38():
@@ -185,9 +186,9 @@ def test_format_accepts_encoding(load_file):
 
 
 def test_stream(get_stream):
-    stream = get_stream("stream.sql")
-    p = sqlparse.parse(stream)[0]
-    assert p.get_type() == 'INSERT'
+    with get_stream("stream.sql") as stream:
+        p = sqlparse.parse(stream)[0]
+        assert p.get_type() == 'INSERT'
 
 
 def test_issue90():
@@ -237,9 +238,9 @@ def test_null_with_as():
 
 def test_issue190_open_file(filepath):
     path = filepath('stream.sql')
-    stream = open(path)
-    p = sqlparse.parse(stream)[0]
-    assert p.get_type() == 'INSERT'
+    with open(path) as stream:
+        p = sqlparse.parse(stream)[0]
+        assert p.get_type() == 'INSERT'
 
 
 def test_issue193_splitting_function():
@@ -316,7 +317,52 @@ def test_token_next_doesnt_ignore_skip_cm():
     assert tok.value == 'select'
 
 
-def test_issue284_as_grouping():
-    sql = 'SELECT x AS'
-    p = sqlparse.parse(sql)[0]
-    assert sql == str(p)
+@pytest.mark.parametrize('s', [
+    'SELECT x AS',
+    'AS'
+])
+def test_issue284_as_grouping(s):
+    p = sqlparse.parse(s)[0]
+    assert s == str(p)
+
+
+def test_issue315_utf8_by_default():
+    # Make sure the lexer can handle utf-8 string by default correctly
+    # digest = '齐天大圣.カラフルな雲.사랑해요'
+    # The digest contains Chinese, Japanese and Korean characters
+    # All in 'utf-8' encoding.
+    digest = (
+        '\xe9\xbd\x90\xe5\xa4\xa9\xe5\xa4\xa7\xe5\x9c\xa3.'
+        '\xe3\x82\xab\xe3\x83\xa9\xe3\x83\x95\xe3\x83\xab\xe3\x81\xaa\xe9'
+        '\x9b\xb2.'
+        '\xec\x82\xac\xeb\x9e\x91\xed\x95\xb4\xec\x9a\x94'
+    )
+    sql = "select * from foo where bar = '{0}'".format(digest)
+    formatted = sqlparse.format(sql, reindent=True)
+    tformatted = "select *\nfrom foo\nwhere bar = '{0}'".format(digest)
+    if PY2:
+        tformatted = tformatted.decode('utf-8')
+    assert formatted == tformatted
+
+
+def test_issue322_concurrently_is_keyword():
+    s = 'CREATE INDEX CONCURRENTLY myindex ON mytable(col1);'
+    p = sqlparse.parse(s)[0]
+
+    assert len(p.tokens) == 12
+    assert p.tokens[0].ttype is T.Keyword.DDL  # CREATE
+    assert p.tokens[2].ttype is T.Keyword      # INDEX
+    assert p.tokens[4].ttype is T.Keyword      # CONCURRENTLY
+    assert p.tokens[4].value == 'CONCURRENTLY'
+    assert isinstance(p.tokens[6], sql.Identifier)
+    assert p.tokens[6].value == 'myindex'
+
+
+@pytest.mark.parametrize('s', [
+    'SELECT @min_price:=MIN(price), @max_price:=MAX(price) FROM shop;',
+    'SELECT @min_price:=MIN(price), @max_price:=MAX(price) FROM shop',
+
+])
+def test_issue359_index_error_assignments(s):
+    sqlparse.parse(s)
+    sqlparse.format(s, strip_comments=True)
diff --git a/tox.ini b/tox.ini
index 494246f..e22c808 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,12 +1,14 @@
 [tox]
+skip_missing_interpreters = True
 envlist =
-    py27,
-    py33,
-    py34,
-    py35,
-    py36,
-    pypy,
-    pypy3,
+    py27
+    py33
+    py34
+    py35
+    py36
+    py37
+    pypy_54
+    pypy33
     flake8
 
 [testenv]
author	Matthias Klose <doko@debian.org>	2018-02-06 17:27:17 +0100
committer	git-ubuntu importer <ubuntu-devel-discuss@lists.ubuntu.com>	2018-02-06 18:10:11 +0000
commit	2232060eacf87c0833316739e97e765df03e7792 (patch)
tree	3c8075a724e6a53f6aac75f8673de16d9ba88e66
parent	da0cdcafe1c50c66fa73fecd52f6cab9faa76a5c (diff)
parent	0abadd07d7b733e1cc9a9079265df5e95a57a461 (diff)