Attachment 125940 Details for Bug 169643 – file.shar

file.shar

file.shar (text/plain), 33.40 KB, created by c.kworr on 2012-07-04 14:00:21 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: c.kworr

Created: 2012-07-04 14:00:21 UTC

Size: 33.40 KB

patch

obsolete

># This is a shell archive.  Save it in a file, remove anything before
># this line, and then unpack it by entering "sh file".  Note, it may
># create directories; files and directories will be owned by you and
># have default permissions.
>#
># This archive contains:
>#
>#	py-mysql2pgsql
>#	py-mysql2pgsql/pkg-plist
>#	py-mysql2pgsql/distinfo
>#	py-mysql2pgsql/Makefile
>#	py-mysql2pgsql/files
>#	py-mysql2pgsql/files/extra-patch
>#	py-mysql2pgsql/pkg-descr
>#
>echo c - py-mysql2pgsql
>mkdir -p py-mysql2pgsql > /dev/null 2>&1
>echo x - py-mysql2pgsql/pkg-plist
>sed 's/^X//' >py-mysql2pgsql/pkg-plist << '48968953d7ef2d4a4eb975f071270e24'
>Xbin/py-mysql2pgsql
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/EGG-INFO/PKG-INFO
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/EGG-INFO/SOURCES.txt
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/EGG-INFO/dependency_links.txt
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/EGG-INFO/not-zip-safe
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/EGG-INFO/requires.txt
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/EGG-INFO/scripts/py-mysql2pgsql
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/EGG-INFO/top_level.txt
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/__init__.py
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/__init__.pyc
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/__init__.pyo
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/__init__.py
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/__init__.pyc
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/__init__.pyo
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/config.py
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/config.pyc
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/config.pyo
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/converter.py
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/converter.pyc
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/converter.pyo
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/errors.py
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/errors.pyc
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/errors.pyo
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/mysql_reader.py
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/mysql_reader.pyc
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/mysql_reader.pyo
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/postgres_db_writer.py
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/postgres_db_writer.pyc
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/postgres_db_writer.pyo
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/postgres_file_writer.py
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/postgres_file_writer.pyc
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/postgres_file_writer.pyo
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/postgres_writer.py
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/postgres_writer.pyc
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/postgres_writer.pyo
>X%%EXTRA%%%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/writer.py
>X%%EXTRA%%%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/writer.pyc
>X%%EXTRA%%%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib/writer.pyo
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/mysql2pgsql.py
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/mysql2pgsql.pyc
>X%%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/mysql2pgsql.pyo
>X@dirrm %%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/EGG-INFO/scripts
>X@dirrm %%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/EGG-INFO
>X@dirrm %%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql/lib
>X@dirrm %%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%/mysql2pgsql
>X@dirrm %%PYTHON_SITELIBDIR%%/%%PYEASYINSTALL_EGG%%
>48968953d7ef2d4a4eb975f071270e24
>echo x - py-mysql2pgsql/distinfo
>sed 's/^X//' >py-mysql2pgsql/distinfo << '8fb20bf151eb4573a259e9807f986359'
>XSHA256 (postgresql/mysql2pgsql/v0.1.2) = 76354d3533adb70757cd1861f7fb68264e0121cd47b32d6050702bb0951b181f
>XSIZE (postgresql/mysql2pgsql/v0.1.2) = 105971
>8fb20bf151eb4573a259e9807f986359
>echo x - py-mysql2pgsql/Makefile
>sed 's/^X//' >py-mysql2pgsql/Makefile << '1d275f4ae6d6996bd614d0f92514c394'
>X# New ports collection makefile for:	py-mysql2pgsql
>X# Date created:				29 May 2011
>X# Whom:					Volodymyr Kostyrko <c.kworr@gmail.com>
>X# vim:ts=8
>X#
>X# $FreeBSD$
>X#
>X
>XPORTNAME=	mysql2pgsql
>XPORTVERSION=	0.1.2
>XCATEGORIES=	databases python
>XMASTER_SITES=	https://nodeload.github.com/${GITHUB_USER}/py-${PORTNAME}/tarball/
>XPKGNAMEPREFIX=	${PYTHON_PKGNAMEPREFIX}
>XPKGNAMESUFFIX?=	${EXTRA_SUFFIX}
>XDISTNAME=	v${PORTVERSION}
>XEXTRACT_SUFX=
>XDIST_SUBDIR=	postgresql/${PORTNAME}
>X
>XMAINTAINER=	c.kworr@gmail.com
>XCOMMENT=	Tool for migrating/converting from mysql to postgresql
>X
>XLICENSE=	MIT # ?dunno, looks like MIT
>XLICENSE_FILE=	${WRKSRC}/LICENSE
>X
>XRUN_DEPENDS=	${PYTHON_PKGNAMEPREFIX}MySQLdb>0:${PORTSDIR}/databases/py-MySQLdb \
>X		${PYTHON_PKGNAMEPREFIX}psycopg2>0:${PORTSDIR}/databases/py-psycopg2 \
>X		${PYTHON_PKGNAMEPREFIX}termcolor>0:${PORTSDIR}/devel/py-termcolor \
>X		${PYTHON_PKGNAMEPREFIX}yaml>0:${PORTSDIR}/devel/py-yaml
>X
>XUSE_PYTHON=	yes
>XUSE_PYDISTUTILS=	easy_install
>XPYDISTUTILS_PKGNAME=	py_${PORTNAME}
>XGITHUB_USER=	philipsoutham
>XGITHUB_HASH=	74de1e0
>XWRKSRC=	${WRKDIR}/${GITHUB_USER}-py-${PORTNAME}-${GITHUB_HASH}
>X
>XOPTIONS_DEFINE=	EXTRA
>XEXTRA_DESC=	Extra patches from github repo not tagged separately
>X# Actually includes a lot of my own patches, I'm trying to stuff it to the master branch
>X
>X.include <bsd.port.pre.mk>
>X
>X.if ${PORT_OPTIONS:MEXTRA}
>XPLIST_SUB+=	EXTRA="@comment "
>XEXTRA_PATCHES+=	${FILESDIR}/extra-patch
>XEXTRA_SUFFIX?=	+extra
>X.else
>XPLIST_SUB+=	EXTRA="@comment "
>X.endif
>X
>X.include <bsd.port.post.mk>
>1d275f4ae6d6996bd614d0f92514c394
>echo c - py-mysql2pgsql/files
>mkdir -p py-mysql2pgsql/files > /dev/null 2>&1
>echo x - py-mysql2pgsql/files/extra-patch
>sed 's/^X//' >py-mysql2pgsql/files/extra-patch << '0c3925eddecf298c646bd0c56b7311c4'
>X--- .travis.yml	Thu Jan 01 00:00:00 1970 +0000
>X+++ .travis.yml	Wed Jul 04 12:03:14 2012 +0300
>X@@ -0,0 +1,21 @@
>X+language: python
>X+python:
>X+  - "2.6"
>X+  - "2.7"
>X+install:
>X+  - pip install . --use-mirrors
>X+  - pip install -r requirements.txt --use-mirrors
>X+script: nosetests
>X+before_script:
>X+  - mysql -e 'create database mysql2pgsql;'
>X+  - psql -c 'create database mysql2pgsql;' -U postgres
>X+env:
>X+  - DB=mysql
>X+  - DB=postgres
>X+branches:
>X+  only:
>X+    - master
>X+    - develop
>X+notifications:
>X+  email:
>X+    - philipsoutham@gmail.com
>X--- bin/py-mysql2pgsql	Thu Aug 18 13:23:12 2011 -0700
>X+++ bin/py-mysql2pgsql	Wed Jul 04 12:03:14 2012 +0300
>X@@ -1,54 +1,33 @@
>X #! /usr/bin/env python
>X import os
>X import sys
>X+import argparse
>X import mysql2pgsql
>X from mysql2pgsql.lib.errors import ConfigurationFileInitialized
>X 
>X if __name__ == '__main__':
>X     description = 'Tool for migrating/converting data from mysql to postgresql.'
>X     epilog = 'https://github.com/philipsoutham/py-mysql2pgsql'
>X-    try:
>X-        import argparse
>X-        parser = argparse.ArgumentParser(
>X-            description=description,
>X-            epilog=epilog)
>X-        parser.add_argument(
>X-            '-v', '--verbose',
>X-            action='store_true',
>X-            help='Show progress of data migration.'
>X-            )
>X-        parser.add_argument(
>X-            '-f', '--file',
>X-            default='mysql2pgsql.yml',
>X-            help='Location of configuration file (default: %(default)s). If none exists at that path, one will be created for you.',
>X-            )
>X-        parser.add_argument(
>X-            '-V', '--version',
>X-            action='store_true',
>X-            help='Print version and exit.'
>X-            )
>X-        options = parser.parse_args()
>X-    except ImportError:
>X-        import optparse
>X-        parser = optparse.OptionParser(
>X-            description=description,
>X-            epilog=epilog)
>X-        parser.add_argument(
>X-            '-v', '--verbose',
>X-            action='store_true',
>X-            help='Show progress of data migration.'
>X-            )
>X-        parser.add_argument(
>X-            '-f', '--file',
>X-            default='mysql2pgsql.yml',
>X-            help='Location of configuration file (default: %default). If none exists at that path, one will be created for you.',
>X-            )
>X-        parser.add_argument(
>X-            '-V', '--version',
>X-            action='store_true',
>X-            help='Print version and exit.'
>X-            )
>X-        options, args = parser.parse_args()
>X+
>X+    parser = argparse.ArgumentParser(
>X+        description=description,
>X+        epilog=epilog)
>X+    parser.add_argument(
>X+        '-v', '--verbose',
>X+        action='store_true',
>X+        help='Show progress of data migration.'
>X+        )
>X+    parser.add_argument(
>X+        '-f', '--file',
>X+        default='mysql2pgsql.yml',
>X+        help='Location of configuration file (default: %(default)s). If none exists at that path, one will be created for you.',
>X+        )
>X+    parser.add_argument(
>X+        '-V', '--version',
>X+        action='store_true',
>X+        help='Print version and exit.'
>X+        )
>X+    options = parser.parse_args()
>X 
>X     if options.version:
>X         # Someone wants to know the version, print and exit
>X--- mysql2pgsql/lib/__init__.py	Thu Aug 18 13:23:12 2011 -0700
>X+++ mysql2pgsql/lib/__init__.py	Wed Jul 04 12:03:14 2012 +0300
>X@@ -5,7 +5,7 @@
>X 
>X from .mysql_reader import MysqlReader
>X try:
>X-    from termcolor import colored, cprint
>X+    from termcolor import cprint
>X except ImportError:
>X     pass
>X 
>X@@ -89,4 +89,3 @@
>X         else:
>X             return f(*args, **kwargs)
>X     return decorated_function
>X-
>X--- mysql2pgsql/lib/config.py	Thu Aug 18 13:23:12 2011 -0700
>X+++ mysql2pgsql/lib/config.py	Wed Jul 04 12:03:14 2012 +0300
>X@@ -2,7 +2,7 @@
>X 
>X import os.path
>X 
>X-from yaml import load, dump
>X+from yaml import load
>X 
>X try:
>X     from yaml import CLoader as Loader, CDumper as Dumper
>X@@ -17,6 +17,7 @@
>X     def __init__(self, config_file_path):
>X         self.options = load(open(config_file_path))
>X 
>X+
>X class Config(ConfigBase):
>X     def __init__(self, config_file_path, generate_if_not_found=True):
>X         if not os.path.isfile(config_file_path):
>X@@ -34,7 +35,7 @@
>X     def reset_configfile(self, file_path):
>X         with open(file_path, 'w') as f:
>X             f.write(CONFIG_TEMPLATE)
>X-        
>X+
>X CONFIG_TEMPLATE = """
>X # if a socket is specified we will use that
>X # if tcp is chosen you can use compression
>X--- mysql2pgsql/lib/converter.py	Thu Aug 18 13:23:12 2011 -0700
>X+++ mysql2pgsql/lib/converter.py	Wed Jul 04 12:03:14 2012 +0300
>X@@ -20,7 +20,9 @@
>X             print_start_table('>>>>>>>>>> STARTING <<<<<<<<<<\n\n')
>X 
>X         tables = [t for t in (t for t in self.reader.tables if t.name not in self.exclude_tables) if not self.only_tables or t.name in self.only_tables]
>X-
>X+        if self.only_tables:
>X+            tables.sort(key=lambda t: self.only_tables.index(t.name))
>X+        
>X         if not self.supress_ddl:
>X             if self.verbose:
>X                 print_start_table('START CREATING TABLES')
>X@@ -57,12 +59,13 @@
>X 
>X             for table in tables:
>X                 self.writer.write_indexes(table)
>X+
>X+            for table in tables:
>X                 self.writer.write_constraints(table)
>X 
>X             if self.verbose:
>X                 print_start_table('DONE CREATING INDEXES AND CONSTRAINTS')
>X 
>X-
>X         if self.verbose:
>X             print_start_table('\n\n>>>>>>>>>> FINISHED <<<<<<<<<<')
>X 
>X--- mysql2pgsql/lib/mysql_reader.py	Thu Aug 18 13:23:12 2011 -0700
>X+++ mysql2pgsql/lib/mysql_reader.py	Wed Jul 04 12:03:14 2012 +0300
>X@@ -11,7 +11,8 @@
>X re_column_precision = re.compile(r'$(\d+),(\d+)$')
>X re_key_1 = re.compile(r'CONSTRAINT `(\w+)` FOREIGN KEY $`(\w+)`$ REFERENCES `(\w+)` $`(\w+)`$')
>X re_key_2 = re.compile(r'KEY `(\w+)` $(.*)$')
>X-re_key_3 = re.compile(r'PRIMARY KEY .*$(.*)$')
>X+re_key_3 = re.compile(r'PRIMARY KEY $(.*)$')
>X+
>X 
>X class DB:
>X     """
>X@@ -21,6 +22,7 @@
>X     helper functions.
>X     """
>X     conn = None
>X+
>X     def __init__(self, options):
>X         args = {
>X             'user': options.get('username', 'root'),
>X@@ -42,7 +44,7 @@
>X         self.options = args
>X 
>X     def connect(self):
>X-        self.conn =  MySQLdb.connect(**self.options)
>X+        self.conn = MySQLdb.connect(**self.options)
>X 
>X     def close(self):
>X         self.conn.close()
>X@@ -86,37 +88,43 @@
>X 
>X         def _convert_type(self, data_type):
>X             """Normalize MySQL `data_type`"""
>X-            if 'varchar' in data_type:
>X+            if data_type.startswith('varchar'):
>X                 return 'varchar'
>X-            elif 'char' in data_type:
>X+            elif data_type.startswith('char'):
>X                 return 'char'
>X             elif data_type in ('bit(1)', 'tinyint(1)', 'tinyint(1) unsigned'):
>X                 return 'boolean'
>X-            elif re.search(r'smallint.* unsigned', data_type) or 'mediumint' in data_type:
>X+            elif re.search(r'^smallint.* unsigned', data_type) or data_type.startswith('mediumint'):
>X                 return 'integer'
>X-            elif 'smallint' in data_type:
>X+            elif data_type.startswith('smallint'):
>X                 return 'tinyint'
>X-            elif 'tinyint' in data_type or 'year(' in data_type:
>X+            elif data_type.startswith('tinyint') or data_type.startswith('year('):
>X                 return 'tinyint'
>X-            elif 'bigint' in data_type and 'unsigned' in data_type:
>X+            elif data_type.startswith('bigint') and 'unsigned' in data_type:
>X                 return 'numeric'
>X-            elif re.search(r'int.* unsigned', data_type) or\
>X+            elif re.search(r'^int.* unsigned', data_type) or\
>X                     ('bigint' in data_type and 'unsigned' not in data_type):
>X                 return 'bigint'
>X-            elif 'int' in data_type:
>X+            elif data_type.startswith('int'):
>X                 return 'integer'
>X-            elif 'float' in data_type:
>X+            elif data_type.startswith('float'):
>X                 return 'float'
>X-            elif 'decimal' in data_type:
>X+            elif data_type.startswith('decimal'):
>X                 return 'decimal'
>X-            elif 'double' in data_type:
>X+            elif data_type.startswith('double'):
>X                 return 'double precision'
>X             else:
>X                 return data_type
>X 
>X         def _load_columns(self):
>X             fields = []
>X-            for res in self.reader.db.query('EXPLAIN `%s`' % self.name):
>X+            for row in self.reader.db.query('EXPLAIN `%s`' % self.name):
>X+                res = ()
>X+                for field in row:
>X+                  if type(field) == unicode:
>X+                    res += field.encode('utf8'),
>X+                  else:
>X+                    res += field,
>X                 length_match = re_column_length.search(res[1])
>X                 precision_match = re_column_precision.search(res[1])
>X                 length = length_match.group(1) if length_match else \
>X@@ -138,7 +146,7 @@
>X                 res = self.reader.db.query('SELECT MAX(`%s`) FROM `%s`;' % (field['name'], self.name), one=True)
>X                 field['maxval'] = int(res[0]) if res[0] else 0
>X             return fields
>X-                
>X+
>X         def _load_indexes(self):
>X             explain = self.reader.db.query('SHOW CREATE TABLE `%s`' % self.name, one=True)
>X             explain = explain[1]
>X@@ -164,7 +172,7 @@
>X                 match_data = re_key_3.search(line)
>X                 if match_data:
>X                     index['primary'] = True
>X-                    index['columns'] = [col.replace('`', '') for col in match_data.group(1).split(',')]
>X+                    index['columns'] = [re.sub(r'$\d+$', '', col.replace('`', '')) for col in match_data.group(1).split(',')]
>X                     self._indexes.append(index)
>X                     continue
>X 
>X@@ -189,7 +197,7 @@
>X             return 'SELECT %(column_names)s FROM `%(table_name)s`' % {
>X                 'table_name': self.name,
>X                 'column_names': ', '. join(("`%s`" % c['name']) for c in self.columns)}
>X-                
>X+
>X     def __init__(self, options):
>X         self.db = DB(options)
>X 
>X--- mysql2pgsql/lib/postgres_db_writer.py	Thu Aug 18 13:23:12 2011 -0700
>X+++ mysql2pgsql/lib/postgres_db_writer.py	Wed Jul 04 12:03:14 2012 +0300
>X@@ -1,15 +1,14 @@
>X from __future__ import with_statement, absolute_import
>X 
>X-import sys
>X import time
>X from contextlib import closing
>X 
>X import psycopg2
>X-from psycopg2.extensions import QuotedString
>X 
>X from . import print_row_progress, status_logger
>X from .postgres_writer import PostgresWriter
>X 
>X+
>X class PostgresDbWriter(PostgresWriter):
>X     """Class used to stream DDL and/or data
>X     from a MySQL server to a PostgreSQL.
>X@@ -53,7 +52,7 @@
>X                 try:
>X                     return '%s\n' % ('\t'.join(row))
>X                 except UnicodeDecodeError:
>X-                    return '%s\n' % ('\t'.join(row)).decode('utf-8')
>X+                    return '%s\n' % ('\t'.join(r.decode('utf8') for r in row))
>X             finally:
>X                 if self.verbose:
>X                     if (self.idx % 20000) == 0:
>X@@ -69,8 +68,8 @@
>X         def read(self, *args, **kwargs):
>X             return self.readline(*args, **kwargs)
>X 
>X-
>X     def __init__(self, db_options, verbose=False):
>X+        super(PostgresDbWriter, self).__init__()
>X         self.verbose = verbose
>X         self.db_options = {
>X             'host': db_options['hostname'],
>X@@ -80,7 +79,7 @@
>X             'user': db_options['username'],
>X             }
>X         if ':' in db_options['database']:
>X-            self.db_options['database'], self.schema  = self.db_options['database'].split(':')
>X+            self.db_options['database'], self.schema = self.db_options['database'].split(':')
>X         else:
>X             self.schema = None
>X         self.open()
>X@@ -115,7 +114,7 @@
>X                           table=table_name,
>X                           columns=columns
>X                           )
>X-    
>X+
>X         self.conn.commit()
>X 
>X     def close(self):
>X@@ -129,13 +128,13 @@
>X     @status_logger
>X     def truncate(self, table):
>X         """Send DDL to truncate the specified `table`
>X-        
>X+
>X         :Parameters:
>X           - `table`: an instance of a :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader.Table` object that represents the table to read/write.
>X 
>X         Returns None
>X         """
>X-        truncate_sql, serial_key_sql = super(self.__class__, self).truncate(table)
>X+        truncate_sql, serial_key_sql = super(PostgresDbWriter, self).truncate(table)
>X         self.execute(truncate_sql)
>X         if serial_key_sql:
>X             self.execute(serial_key_sql)
>X@@ -149,10 +148,10 @@
>X 
>X         Returns None
>X         """
>X-        table_sql, serial_key_sql = super(self.__class__, self).write_table(table)
>X+        table_sql, serial_key_sql = super(PostgresDbWriter, self).write_table(table)
>X         for sql in serial_key_sql + table_sql:
>X             self.execute(sql)
>X-        
>X+
>X     @status_logger
>X     def write_indexes(self, table):
>X         """Send DDL to create the specified `table` indexes
>X@@ -162,7 +161,7 @@
>X 
>X         Returns None
>X         """
>X-        index_sql = super(self.__class__, self).write_indexes(table)
>X+        index_sql = super(PostgresDbWriter, self).write_indexes(table)
>X         for sql in index_sql:
>X             self.execute(sql)
>X 
>X@@ -175,7 +174,7 @@
>X 
>X         Returns None
>X         """
>X-        constraint_sql = super(self.__class__, self).write_constraints(table)
>X+        constraint_sql = super(PostgresDbWriter, self).write_constraints(table)
>X         for sql in constraint_sql:
>X             self.execute(sql)
>X 
>X--- mysql2pgsql/lib/postgres_file_writer.py	Thu Aug 18 13:23:12 2011 -0700
>X+++ mysql2pgsql/lib/postgres_file_writer.py	Wed Jul 04 12:03:14 2012 +0300
>X@@ -1,16 +1,13 @@
>X from __future__ import absolute_import
>X 
>X import time
>X-import sys
>X 
>X-from cStringIO import StringIO
>X-
>X-from psycopg2.extensions import QuotedString
>X 
>X from .postgres_writer import PostgresWriter
>X 
>X from . import print_row_progress, status_logger
>X 
>X+
>X class PostgresFileWriter(PostgresWriter):
>X     """Class used to ouput the PostgreSQL
>X     compatable DDL and/or data to the specified
>X@@ -19,10 +16,12 @@
>X     :Parameters:
>X       - `output_file`: the output :py:obj:`file` to send the DDL and/or data
>X       - `verbose`: whether or not to log progress to :py:obj:`stdout`
>X-    
>X+
>X     """
>X     verbose = None
>X+
>X     def __init__(self, output_file, verbose=False):
>X+        super(PostgresFileWriter, self).__init__()
>X         self.verbose = verbose
>X         self.f = output_file
>X         self.f.write("""
>X@@ -42,7 +41,7 @@
>X 
>X         Returns None
>X         """
>X-        truncate_sql, serial_key_sql = super(self.__class__, self).truncate(table)
>X+        truncate_sql, serial_key_sql = super(PostgresFileWriter, self).truncate(table)
>X         self.f.write("""
>X -- TRUNCATE %(table_name)s;
>X %(truncate_sql)s
>X@@ -63,9 +62,9 @@
>X 
>X         Returns None
>X         """
>X-        table_sql, serial_key_sql = super(self.__class__, self).write_table(table)
>X+        table_sql, serial_key_sql = super(PostgresFileWriter, self).write_table(table)
>X         if serial_key_sql:
>X-            self.f.write(""" 
>X+            self.f.write("""
>X %(serial_key_sql)s
>X """ % {
>X     'serial_key_sql': '\n'.join(serial_key_sql)
>X@@ -88,7 +87,7 @@
>X 
>X         Returns None
>X         """
>X-        self.f.write('\n'.join(super(self.__class__, self).write_indexes(table)))
>X+        self.f.write('\n'.join(super(PostgresFileWriter, self).write_indexes(table)))
>X 
>X     @status_logger
>X     def write_constraints(self, table):
>X@@ -99,7 +98,7 @@
>X 
>X         Returns None
>X         """
>X-        self.f.write('\n'.join(super(self.__class__, self).write_constraints(table)))
>X+        self.f.write('\n'.join(super(PostgresFileWriter, self).write_constraints(table)))
>X 
>X     @status_logger
>X     def write_contents(self, table, reader):
>X@@ -119,7 +118,7 @@
>X 
>X         f_write("""
>X --
>X--- Data for Name: %(table_name)s; Type: TABLE DATA; 
>X+-- Data for Name: %(table_name)s; Type: TABLE DATA;
>X --
>X 
>X COPY "%(table_name)s" (%(column_names)s) FROM stdin;
>X@@ -131,22 +130,22 @@
>X             start_time = tt()
>X             prev_val_len = 0
>X             prev_row_count = 0
>X-        for i, row in enumerate(reader.read(table)):
>X+        for i, row in enumerate(reader.read(table), 1):
>X             row = list(row)
>X             pr(table, row)
>X             try:
>X-                f_write('%s\n' % ('\t'.join(row)))
>X+                f_write(u'%s\n' % (u'\t'.join(row)))
>X             except UnicodeDecodeError:
>X-                f_write('%s\n' % ('\t'.join(row)).decode('utf-8'))
>X+                f_write(u'%s\n' % (u'\t'.join(r.decode('utf-8') for r in row)))
>X             if verbose:
>X-                if ((i + 1) % 20000) == 0:
>X+                if (i % 20000) == 0:
>X                     now = tt()
>X                     elapsed = now - start_time
>X-                    val = '%.2f rows/sec [%s] ' % (((i + 1) - prev_row_count) / elapsed, (i + 1))
>X+                    val = '%.2f rows/sec [%s] ' % ((i - prev_row_count) / elapsed, i)
>X                     print_row_progress('%s%s' % (("\b" * prev_val_len), val))
>X                     prev_val_len = len(val) + 3
>X                     start_time = now
>X-                    prev_row_count = i + 1
>X+                    prev_row_count = i
>X 
>X         f_write("\\.\n\n")
>X         if verbose:
>X--- mysql2pgsql/lib/postgres_writer.py	Thu Aug 18 13:23:12 2011 -0700
>X+++ mysql2pgsql/lib/postgres_writer.py	Wed Jul 04 12:03:14 2012 +0300
>X@@ -6,18 +6,21 @@
>X 
>X from psycopg2.extensions import QuotedString, Binary, AsIs
>X 
>X-from .writer import Writer
>X 
>X-
>X-class PostgresWriter(Writer):
>X+class PostgresWriter(object):
>X     """Base class for :py:class:`mysql2pgsql.lib.postgres_file_writer.PostgresFileWriter`
>X     and :py:class:`mysql2pgsql.lib.postgres_db_writer.PostgresDbWriter`.
>X     """
>X+    def __init__(self):
>X+        self.column_types = {}
>X+
>X     def column_description(self, column):
>X         return '"%s" %s' % (column['name'], self.column_type_info(column))
>X 
>X     def column_type(self, column):
>X-        return self.column_type_info(column).split(" ")[0]
>X+        hash_key = hash(frozenset(column.items()))
>X+        self.column_types[hash_key] = self.column_type_info(column).split(" ")[0]
>X+        return self.column_types[hash_key]
>X 
>X     def column_type_info(self, column):
>X         """
>X@@ -26,15 +29,14 @@
>X             return 'integer DEFAULT nextval(\'%s_%s_seq\'::regclass) NOT NULL' % (
>X                    column['table_name'], column['name'])
>X 
>X-        
>X         null = "" if column['null'] else " NOT NULL"
>X-        
>X+
>X         def get_type(column):
>X             """This in conjunction with :py:class:`mysql2pgsql.lib.mysql_reader.MysqlReader._convert_type`
>X             determines the PostgreSQL data type. In my opinion this is way too fugly, will need
>X             to refactor one day.
>X             """
>X-            def t(v): return not v == None
>X+            t = lambda v: not v == None
>X             default = (' DEFAULT %s' % QuotedString(column['default']).getquoted()) if t(column['default']) else None
>X 
>X             if column['type'] == 'char':
>X@@ -74,11 +76,13 @@
>X                 default = None
>X                 return default, 'date'
>X             elif column['type'] == 'timestamp':
>X-                if "CURRENT_TIMESTAMP" in column['default']:
>X+                if column['default'] == None:
>X+                    default = None
>X+                elif "CURRENT_TIMESTAMP" in column['default']:
>X                     default = ' DEFAULT CURRENT_TIMESTAMP'
>X-                if "0000-00-00 00:00" in  column['default']:
>X+                elif "0000-00-00 00:00" in  column['default']:
>X                     default = " DEFAULT '1970-01-01 00:00'"
>X-                if "0000-00-00 00:00:00" in column['default']:
>X+                elif "0000-00-00 00:00:00" in column['default']:
>X                     default = " DEFAULT '1970-01-01 00:00:00'"
>X                 return default, 'timestamp without time zone'
>X             elif column['type'] == 'time':
>X@@ -90,8 +94,8 @@
>X                 return default, 'text'
>X             elif re.search(r'^enum', column['type']):
>X                 default = (' %s::character varying' % default) if t(default) else None
>X-                enum = re.sub(r'enum|$|$', '', column['type'])
>X-                max_enum_size = max([(len(e) - 2) for e in enum.split(',')])
>X+                enum = re.sub(r'^enum$|$$', '', column['type'])
>X+                max_enum_size = max([len(e.replace("''", "'")) for e in enum.split("','")])
>X                 return default, ' character varying(%s) check(%s in (%s))' % (max_enum_size, column['name'], enum)
>X             elif 'bit(' in column['type']:
>X                 return ' DEFAULT %s' % column['default'].upper() if column['default'] else column['default'], 'varbit(%s)' % re.search(r'$(\d+)$', column['type']).group(1)
>X@@ -111,14 +115,15 @@
>X         sending to PostgreSQL via the copy command
>X         """
>X         for index, column in enumerate(table.columns):
>X-            column_type = self.column_type(column)
>X+            hash_key = hash(frozenset(column.items()))
>X+            column_type = self.column_types[hash_key] if hash_key in self.column_types else self.column_type(column)
>X             if row[index] == None and ('timestamp' not in column_type or not column['default']):
>X                 row[index] = '\N'
>X             elif row[index] == None and column['default']:
>X                 row[index] = '1970-01-01 00:00:00'
>X             elif 'bit' in column_type:
>X                 row[index] = bin(ord(row[index]))[2:]
>X-            elif row[index].__class__ in (str, unicode):
>X+            elif isinstance(row[index], (str, unicode, basestring)):
>X                 if column_type == 'bytea':
>X                     row[index] = Binary(row[index]).getquoted()[1:-8] if row[index] else row[index]
>X                 elif 'text[' in column_type:
>X@@ -126,10 +131,11 @@
>X                 else:
>X                     row[index] = row[index].replace('\\', r'\\').replace('\n', r'\n').replace('\t', r'\t').replace('\r', r'\r').replace('\0', '')
>X             elif column_type == 'boolean':
>X-                row[index] = 't' if row[index] == 1 else 'f' if row[index] == 0 else row[index]
>X-            elif row[index].__class__ in (date, datetime):
>X+                # We got here because you used a tinyint(1), if you didn't want a bool, don't use that type
>X+                row[index] = 't' if row[index] not in (None, 0) else 'f' if row[index] == 0 else row[index]
>X+            elif isinstance(row[index], (date, datetime)):
>X                 row[index] = row[index].isoformat()
>X-            elif row[index].__class__ is timedelta:
>X+            elif isinstance(row[index], timedelta):
>X                 row[index] = datetime.utcfromtimestamp(row[index].total_seconds()).time().isoformat()
>X             else:
>X                 row[index] = AsIs(row[index]).getquoted()
>X@@ -149,7 +155,6 @@
>X             columns.write('  %s,\n' % self.column_description(column))
>X         return primary_keys, serial_key, maxval, columns.getvalue()[:-2]
>X 
>X-
>X     def truncate(self, table):
>X         serial_key = None
>X         maxval = None
>X@@ -182,7 +187,7 @@
>X             serial_key_sql.append('SELECT pg_catalog.setval(%s, %s, true);' % (QuotedString(serial_key_seq).getquoted(), maxval))
>X 
>X         table_sql.append('DROP TABLE IF EXISTS "%s" CASCADE;' % table.name)
>X-        table_sql.append('CREATE TABLE "%s" (\n%s\n)\nWITHOUT OIDS;' % (table.name, columns))
>X+        table_sql.append('CREATE TABLE "%s" (\n%s\n)\nWITHOUT OIDS;' % (table.name.encode('utf8'), columns))
>X         return (table_sql, serial_key_sql)
>X 
>X     def write_indexes(self, table):
>X@@ -191,7 +196,7 @@
>X         if primary_index:
>X             index_sql.append('ALTER TABLE "%(table_name)s" ADD CONSTRAINT "%(index_name)s_pkey" PRIMARY KEY(%(column_names)s);' % {
>X                     'table_name': table.name,
>X-                    'index_name': '%s_%s' % (table.name, '_'.join(primary_index[0]['columns'])),
>X+                    'index_name': '%s_%s' % (table.name, '_'.join(re.sub('[\W]+', '', c) for c in primary_index[0]['columns'])),
>X                     'column_names': ', '.join('"%s"' % col for col in primary_index[0]['columns']),
>X                     })
>X         for index in table.indexes:
>X@@ -206,7 +211,7 @@
>X                     'table_name': table.name,
>X                     'column_names': ', '.join('"%s"' % col for col in index['columns']),
>X                     })
>X-        
>X+
>X         return index_sql
>X 
>X     def write_constraints(self, table):
>X--- mysql2pgsql/lib/writer.py	Thu Aug 18 13:23:12 2011 -0700
>X+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
>X@@ -1,2 +0,0 @@
>X-class Writer(object):
>X-    pass
>X--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
>X+++ requirements.txt	Wed Jul 04 12:03:14 2012 +0300
>X@@ -0,0 +1,5 @@
>X+mysql-python
>X+psycopg2
>X+pyyaml
>X+termcolor
>X+argparse
>X--- setup.py	Thu Aug 18 13:23:12 2011 -0700
>X+++ setup.py	Wed Jul 04 12:03:14 2012 +0300
>X@@ -5,6 +5,7 @@
>X     'mysql-python>=1.2.3', 
>X     'psycopg2>=2.4.2',
>X     'pyyaml>=3.10.0',
>X+    'argparse',
>X ]
>X 
>X if os.name == 'posix':
>X--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
>X+++ tests/mysql2pgsql-test.yml	Wed Jul 04 12:03:14 2012 +0300
>X@@ -0,0 +1,38 @@
>X+
>X+# if a socket is specified we will use that
>X+# if tcp is chosen you can use compression
>X+mysql:
>X+ hostname: 127.0.0.1
>X+ port: 3306
>X+ socket:
>X+ username: root
>X+ password: 
>X+ database: mysql2pgsql
>X+ compress: false
>X+destination:
>X+ # if file is given, output goes to file, else postgres
>X+ file: 
>X+ postgres:
>X+  hostname: 127.0.0.1
>X+  port: 5432
>X+  username: postgres
>X+  password: 
>X+  database: mysql2pgsql
>X+
>X+# if tables is given, only the listed tables will be converted.  leave empty to convert all tables.
>X+#only_tables:
>X+#- table1
>X+#- table2
>X+# if exclude_tables is given, exclude the listed tables from the conversion.
>X+#exclude_tables:
>X+#- table3
>X+#- table4
>X+
>X+# if supress_data is true, only the schema definition will be exported/migrated, and not the data
>X+supress_data: false
>X+
>X+# if supress_ddl is true, only the data will be exported/imported, and not the schema
>X+supress_ddl: false
>X+
>X+# if force_truncate is true, forces a table truncate before table loading
>X+force_truncate: false
>X--- tests/schema.sql	Thu Aug 18 13:23:12 2011 -0700
>X+++ tests/schema.sql	Wed Jul 04 12:03:14 2012 +0300
>X@@ -718,7 +718,7 @@
>X 
>X -- SPLIT
>X 
>X-INSERT INTO `type_conversion_test_2` (`tct_1_id`, `foo`) VALUES (2, 'baz');
>X+INSERT INTO `type_conversion_test_2` (`tct_1_id`, `foo`) VALUES (2, 'special characters:ÄÄÅ¾Å¡Ä');
>X 
>X -- SPLIT
>X 
>X--- tests/test_reader.py	Thu Aug 18 13:23:12 2011 -0700
>X+++ tests/test_reader.py	Wed Jul 04 12:03:14 2012 +0300
>X@@ -31,7 +31,7 @@
>X             }
>X 
>X         if self.options.get('password', None):
>X-            self.args['passwd'] = self.options.get('password', None),
>X+            self.args['passwd'] = self.options.get('password', None)
>X 
>X         if self.options.get('socket', None):
>X             self.args['unix_socket'] = self.options['socket']
>0c3925eddecf298c646bd0c56b7311c4
>echo x - py-mysql2pgsql/pkg-descr
>sed 's/^X//' >py-mysql2pgsql/pkg-descr << '4b6a2462b80b0f47d9fc1e47e66fc6ee'
>XTool for migrating/converting from mysql to postgresql.
>X
>XWWW: http://packages.python.org/py-mysql2pgsql/
>4b6a2462b80b0f47d9fc1e47e66fc6ee
>exit

Actions: View

Attachments on bug 169643: 125940 | 125941