#!/usr/bin/env python
# Copyright (c) 2011 Seth Davis http://www.curiasolutions.com/
# s3put is Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#
import sys, os, time, datetime, getopt, threading, signal
import boto

__version__ = '0.5.1'

usage_string = """
boto-rsync v0.5.1
Copyright (c) 2011 Seth Davis
http://github.com/seedifferently/boto_rsync

SYNOPSIS
    boto-rsync is a rough adaptation of boto's s3put script which has been
    reengineered to more closely mimic rsync. Its goal is to provide a familiar
    rsync-like wrapper for boto's S3 and Google Storage interfaces.

    By default, the script works recursively and differences between files are
    checked by comparing file sizes (e.g. rsync's --recursive and --size-only
    options). If the file exists on the destination but its size differs from
    the source, then it will be overwritten (unless the -w option is used).

USAGE
    boto-rsync [OPTIONS] SOURCE DESTINATION
    
    SOURCE and DESTINATION can either be a local path to a directory or specific
    file, a custom S3 or GS URI to a directory or specific key in the format of
    s3://bucketname/path/or/key, a S3 to S3 transfer using two S3 URIs, or
    a GS to GS transfer using two GS URIs.

EXAMPLES
    boto-rsync [OPTIONS] /local/path/ s3://bucketname/remote/path/
 or
    boto-rsync [OPTIONS] gs://bucketname/remote/path/or/key /local/path/
 or
    boto-rsync [OPTIONS] s3://bucketname/ s3://another_bucket/

OPTIONS
    -a/--access_key <key>       Your Access Key ID. If not supplied, boto will
                                use the value of the environment variable
                                AWS_ACCESS_KEY_ID (for S3) or GS_ACCESS_KEY_ID
                                (for GS).
    -s/--secret_key <secret>    Your Secret Access Key. If not supplied, boto
                                will use the value of the environment variable
                                AWS_ACCESS_KEY_ID (for S3) or GS_ACCESS_KEY_ID
                                (for GS).
    -d/--debug <debug_level>    0 means no debug output (default), 1 means
                                normal debug output from boto, and 2 means boto
                                debug output plus request/response output from
                                httplib.
    -r/--reduced                Enable reduced redundancy on files copied to S3.
    -g/--grant <policy>         A canned ACL policy that will be granted on each
                                file transferred to S3/GS. The value provided
                                must be one of the "canned" ACL policies
                                supported by S3/GS: private, public-read,
                                public-read-write (S3 only), or
                                authenticated-read
    -w/--no_overwrite           No files will be overwritten, if the file/key
                                exists on the destination it will be kept. Note
                                that this is not a sync--even if the file has
                                been updated on the source it will not be
                                updated on the destination.
    --ignore_empty              Ignore empty (0-byte) keys/files/directories.
                                This will skip the transferring of empty
                                directories and keys/files whose size is 0.
                                Warning: S3/GS often uses empty keys with
                                special trailing characters to specify
                                directories.
    -p/--preserve_acl           Copy the ACL from the source key to the
                                destination key (only applies in S3/GS to S3/GS
                                transfer mode).
    -e/--encrypt_keys           Enable server-side encryption on files copied
                                to S3 (only applies when S3 is the destination).
    --delete                    Delete extraneous files from destination dirs
                                after the transfer has finished (e.g. rsync's
                                --delete-after).
    -n/--no_op                  No files will be transferred, but informational
                                messages will be printed about what would happen
                                (e.g. rsync's --dry-run).
    -v/--verbose                Print additional informational messages.
"""
def usage():
    """Prints the usage string and exits."""
    print usage_string
    
    sys.exit()

def get_full_path(path):
    """
    Returns a full path with special markers such as "~" and "$USER" expanded.
    """
    path = os.path.expanduser(path)
    path = os.path.expandvars(path)
    if path and path.endswith(os.sep):
        path = os.path.abspath(path) + os.sep
    else:
        path = os.path.abspath(path)
    return path

def convert_bytes(n):
    """Converts byte sizes into human readable forms such as KB/MB/etc."""
    K, M, G, T = 1 << 10, 1 << 20, 1 << 30, 1 << 40
    if   n >= T:
        return '%.1fT' % (float(n) / T)
    elif n >= G:
        return '%.1fG' % (float(n) / G)
    elif n >= M:
        return '%.1fM' % (float(n) / M)
    elif n >= K:
        return '%.1fK' % (float(n) / K)
    else:
        return '%dB' % n

def spinner(event, every):
    """Animates an ASCII spinner."""
    while True:
        if event.isSet():
            sys.stdout.write('\b \b')
            sys.stdout.flush()
            break
        sys.stdout.write('\b\\')
        sys.stdout.flush()
        event.wait(every)
        sys.stdout.write('\b|')
        sys.stdout.flush()
        event.wait(every)
        sys.stdout.write('\b/')
        sys.stdout.flush()
        event.wait(every)
        sys.stdout.write('\b-')
        sys.stdout.flush()
        event.wait(every)

def submit_cb(bytes_so_far, total_bytes):
    """The "progress" callback for file transfers."""
    global speeds
    
    # Setup speed calculation
    if bytes_so_far < 1:
        speeds = []
        speeds.append((bytes_so_far, time.time()))
    # Skip processing if our last process was less than 850ms ago
    elif bytes_so_far != total_bytes and (time.time() - speeds[-1][1]) < .85:
        return
    
    speeds.append((bytes_so_far, time.time()))
    
    # Try to get ~5 seconds of data info for speed calculation
    s1, t1 = speeds[-1]
    for speed in reversed(speeds):
        s2, t2 = speed
        
        if (t1 - t2) > 5:
            break
    
    # Calculate the speed
    if bytes_so_far == total_bytes:
        # Calculate the overall average speed
        seconds = int(round(speeds[-1][1] - speeds[0][1]))
        if seconds < 1:
            seconds = 1
        speed = 1.0 * total_bytes / seconds
    else:
        # Calculate the current average speed
        seconds = t1 - t2
        if seconds < 1:
            seconds = 1
        size = s1 - s2
        speed = 1.0 * size / seconds
    
    # Calculate the duration
    try:
        if bytes_so_far == total_bytes:
            # Calculate time taken
            duration = int(round(speeds[-1][1] - speeds[0][1]))
        else:
            # Calculate remaining time
            duration = int(round((total_bytes - bytes_so_far) / speed))
        duration = str(datetime.timedelta(seconds=duration))
    except ZeroDivisionError:
        duration = '0:00:00'
    
    # Calculate the progress
    try:
        progress = round((1.0 * bytes_so_far / total_bytes) * 100)
    except ZeroDivisionError:
        progress = 100
    
    sys.stdout.write('    %6s of %6s    %3d%%    %6s/s    %7s    \r' % (
      convert_bytes(bytes_so_far), convert_bytes(total_bytes), progress,
      convert_bytes(speed), duration)
      )
    sys.stdout.flush()

def get_key_name(fullpath, prefix):
    """Returns a key compatible name for a file."""
    key_name = fullpath[len(prefix):]
    l = key_name.split(os.sep)
    key_name = '/'.join(l)
    return key_name.lstrip('/')

def signal_handler(signum, frame):
    """Handles signals."""
    global ev
    
    if signum == signal.SIGINT:
        if ev:
            ev.set()
        
        print ''
        sys.exit(0)

def main():
    global speeds, ev
    
    try:
        opts, args = getopt.getopt(
                sys.argv[1:], 'a:s:d:rg:wpei:nvh',
                ['access_key', 'secret_key', 'debug', 'reduced', 'grant',
                 'no_overwrite', 'ignore_empty', 'preserve_acl', 'encrypt_keys',
                 'delete', 'no_op', 'verbose', 'help']
                )
    except:
        usage()
    
    signal.signal(signal.SIGINT, signal_handler)
    ev = None
    cloud_access_key_id = None
    cloud_secret_access_key = None
    total = 0
    debug = 0
    speeds = []
    cb = submit_cb
    num_cb = 10
    quiet = True
    no_op = False
    grant = None
    no_overwrite = False
    ignore_empty = False
    reduced = False
    preserve = False
    encrypt = False
    delete = False
    rename = False
    copy_file = True
    for o, a in opts:
        if o in ('-h', '--help'):
            usage()
            sys.exit()
        if o in ('-a', '--access_key'):
            cloud_access_key_id = a
        if o in ('-b', '--bucket'):
            bucket_name = a
        if o in ('-d', '--debug'):
            debug = int(a)
        if o in ('-g', '--grant'):
            grant = a
        if o in ('-n', '--no_op'):
            no_op = True
        if o in ('-w', '--no_overwrite'):
            no_overwrite = True
        if o in ('--ignore_empty'):
            ignore_empty = True
        if o in ('-p', '--preserve_acl'):
            preserve = True
        if o in ('-e', '--encrypt_keys'):
            encrypt = True
        if o in ('-r', '--reduced'):
            reduced = True
        if o in ('-v', '--verbose'):
            quiet = False
        if o in ('--delete'):
            delete = True
        if o in ('-s', '--secret_key'):
            cloud_secret_access_key = a
    if len(args) != 2:
        print usage()
    
    if not args[0].startswith('s3://') and args[1].startswith('s3://'):
        # Cloud upload sync
        cloud_service = 's3'
        path = get_full_path(args[0])
        cloud_bucket = args[1][5:].split('/')[0]
        cloud_path = args[1][(len(cloud_bucket) + 5):]
        xfer_type = 'upload'
    elif args[0].startswith('s3://') and not args[1].startswith('s3://'):
        # Cloud download sync
        cloud_service = 's3'
        cloud_bucket = args[0][5:].split('/')[0]
        cloud_path = args[0][(len(cloud_bucket) + 5):]
        path = get_full_path(args[1])
        xfer_type = 'download'
    elif not args[0].startswith('gs://') and args[1].startswith('gs://'):
        # Cloud upload sync
        cloud_service = 'gs'
        path = get_full_path(args[0])
        cloud_bucket = args[1][5:].split('/')[0]
        cloud_path = args[1][(len(cloud_bucket) + 5):]
        xfer_type = 'upload'
    elif args[0].startswith('gs://') and not args[1].startswith('gs://'):
        # Cloud download sync
        cloud_service = 'gs'
        cloud_bucket = args[0][5:].split('/')[0]
        cloud_path = args[0][(len(cloud_bucket) + 5):]
        path = get_full_path(args[1])
        xfer_type = 'download'
    elif args[0].startswith('s3://') and args[1].startswith('s3://'):
        # S3 to S3 sync
        cloud_service = 's3'
        cloud_bucket = args[0][5:].split('/')[0]
        cloud_path = args[0][(len(cloud_bucket) + 5):]
        cloud_dest_bucket = args[1][5:].split('/')[0]
        cloud_dest_path = args[1][(len(cloud_dest_bucket) + 5):]
        xfer_type = 'sync'
    elif args[0].startswith('gs://') and args[1].startswith('gs://'):
        # GS to GS sync
        cloud_service = 'gs'
        cloud_bucket = args[0][5:].split('/')[0]
        cloud_path = args[0][(len(cloud_bucket) + 5):]
        cloud_dest_bucket = args[1][5:].split('/')[0]
        cloud_dest_path = args[1][(len(cloud_dest_bucket) + 5):]
        xfer_type = 'sync'
    else:
        usage()
    
    # Cloud paths shouldn't have a leading slash
    cloud_path = cloud_path.lstrip('/')
    
    if xfer_type in ['download', 'upload']:
        if not os.path.isdir(path) and not os.path.split(path)[0]:
            print '\nERROR: %s is not a valid path (does it exist?)' % (path)
            usage()
        elif not cloud_bucket or len(cloud_bucket) < 3:
            print '\nERROR: Bucket name is invalid'
            usage()
    elif xfer_type in ['sync']:
        if not cloud_bucket or len(cloud_bucket) < 3 and \
           not cloud_dest_bucket or len(cloud_dest_bucket) < 3:
            print '\nERROR: Bucket name is invalid'
            usage()
        
        # Cloud paths shouldn't have a leading slash
        cloud_dest_path = cloud_dest_path.lstrip('/')
    
    
    # Connect to Cloud
    if cloud_service == 'gs':
        c = boto.connect_gs(gs_access_key_id=cloud_access_key_id,
                            gs_secret_access_key=cloud_secret_access_key)
    else:
        c = boto.connect_s3(aws_access_key_id=cloud_access_key_id,
                            aws_secret_access_key=cloud_secret_access_key)
    c.debug = debug
    b = c.get_bucket(cloud_bucket)
    if xfer_type in ['sync']:
        b2 = c.get_bucket(cloud_dest_bucket)
    
    if xfer_type == 'upload':
        # Perform cloud "upload"
        
        if os.path.isdir(path):
            # Possible multi file upload
            sys.stdout.write('Scanning for files to transfer...  ')
            sys.stdout.flush()
            
            if cloud_path and not cloud_path.endswith('/'):
                cloud_path += '/'
            
            # Start "spinner" thread
            ev = threading.Event()
            t1 = threading.Thread(target=spinner, args=(ev, 0.25))
            t1.start()
            
            try:
                keys = {}
                for key in b.list(prefix=cloud_path):
                    keys[key.name] = key.size
            except Exception, e:
                raise e
            finally:
                # End "spinner" thread
                ev.set()
                t1.join()
                
                # Clean stdout
                print ''
            
            # "Walk" the directory and upload files
            for root, dirs, files in os.walk(path):
                if files:
                    for file in files:
                        fullpath = os.path.join(root, file)
                        key_name = cloud_path + get_key_name(fullpath, path)
                        file_size = os.path.getsize(fullpath)
                        
                        if file_size == 0:
                            if ignore_empty:
                                if not quiet:
                                    print 'Skipping %s (empty file)' % (
                                        fullpath[len(path):].lstrip('/')
                                        )
                                continue
                        
                        if key_name in keys:
                            if no_overwrite:
                                if not quiet:
                                    print 'Skipping %s (not overwriting)' % (
                                        fullpath[len(path):].lstrip('/')
                                        )
                                continue
                            elif keys[key_name] == file_size or \
                                 key_name.endswith('/'):
                                if not quiet:
                                    print 'Skipping %s (size matches)' % (
                                        fullpath[len(path):].lstrip('/')
                                        )
                                continue
                        
                        print '%s' % fullpath[len(path):].lstrip('/')
                        
                        if not no_op:
                            # Setup callback
                            num_cb = int(file_size ** .25)
                            
                            # Send the file
                            k = b.new_key(key_name)
                            if cloud_service == 'gs':
                                k.set_contents_from_filename(
                                    fullpath, cb=cb, num_cb=num_cb, policy=grant
                                    )
                            else:
                                k.set_contents_from_filename(
                                    fullpath, cb=cb, num_cb=num_cb,
                                    policy=grant, reduced_redundancy=reduced,
                                    encrypt_key=encrypt
                                    )
                            keys[key_name] = file_size
                            
                            # Clean stdout
                            print ''
                
                # Check for empty subdirectories
                elif root != path:
                    if cloud_service == 'gs':
                        key_name = cloud_path + get_key_name(root, path) + \
                                   '_$folder$'
                    else:
                        key_name = cloud_path + get_key_name(root, path) + '/'
                    
                    if ignore_empty:
                        if not quiet:
                            print 'Skipping %s (empty directory)' % (
                                key_name.replace('_$folder$', '/')
                                )
                        continue
                    elif key_name in keys:
                        if no_overwrite:
                            if not quiet:
                                print 'Skipping %s (not overwriting)' % (
                                    key_name.replace('_$folder$', '/')
                                    )
                            continue
                        elif key_name.endswith('/') or \
                             key_name.endswith('_$folder$'):
                            if not quiet:
                                print 'Skipping %s (size matches)' % (
                                    key_name.replace('_$folder$', '/')
                                    )
                            continue
                    
                    print '%s' % os.path.join(root[len(path):], '').lstrip('/')
                    if not no_op:
                        # Setup callback
                        num_cb = 1
                        
                        # Send the directory
                        k = b.new_key(key_name)
                        if cloud_service == 'gs':
                            k.set_contents_from_string(
                                '', cb=cb, num_cb=num_cb, policy=grant
                                )
                        else:
                            k.set_contents_from_string(
                                '', cb=cb, num_cb=num_cb, policy=grant,
                                reduced_redundancy=reduced, encrypt_key=encrypt
                                )
                        keys[key_name] = 0
                        
                        # Clean stdout
                        print ''
            
            # If specified, perform deletes
            if delete:
                if cloud_path and cloud_path in keys:
                    del(keys[cloud_path])
                
                for root, dirs, files in os.walk(path):
                    for file in files:
                        fullpath = os.path.join(root, file)
                        key_name = cloud_path + get_key_name(fullpath, path)
                        if key_name in keys:
                            del(keys[key_name])
                    
                    if root != path:
                        if cloud_service == 'gs':
                            key_name = cloud_path + get_key_name(root, path) + \
                                       '_$folder$'
                        else:
                            key_name = cloud_path + get_key_name(root, path) + \
                                       '/'
                        
                        if key_name in keys:
                            del(keys[key_name])
                
                for key_name, key_size in keys.iteritems():
                    print 'deleting %s' % (
                        key_name[len(cloud_path):].replace('_$folder$', '/')
                        )
                    if not no_op:
                        # Delete the key
                        b.delete_key(key_name)
        
        elif os.path.isfile(path):
            # Single file upload
            if cloud_path and not cloud_path.endswith('/'):
                key_name = cloud_path
            else:
                key_name = cloud_path + os.path.split(path)[1]
            filename = os.path.split(path)[1]
            file_size = os.path.getsize(path)
            
            copy_file = True
            key = b.get_key(key_name)
            
            if file_size == 0:
                if ignore_empty:
                    if not quiet:
                        print 'Skipping %s -> %s (empty file)' % (
                            filename, key_name.split('/')[-1]
                            )
                    copy_file = False
            
            if key:
                if no_overwrite:
                    copy_file = False
                    if not quiet:
                        if filename != key_name.split('/')[-1]:
                            print 'Skipping %s -> %s (not overwriting)' % (
                                filename, key_name.split('/')[-1]
                                )
                        else:
                            print 'Skipping %s (not overwriting)' % filename
                elif key.size == file_size:
                    copy_file = False
                    if not quiet:
                        if filename != key_name.split('/')[-1]:
                            print 'Skipping %s -> %s (size matches)' % (
                                filename, key_name.split('/')[-1]
                                )
                        else:
                            print 'Skipping %s (size matches)' % filename
            
            if copy_file:
                if filename != key_name.split('/')[-1]:
                    print '%s -> %s' % (filename, key_name.split('/')[-1])
                else:
                    print '%s' % filename
                
                if not no_op:
                    # Setup callback
                    num_cb = int(file_size ** .25)
                    
                    # Send the file
                    k = b.new_key(key_name)
                    if cloud_service == 'gs':
                        k.set_contents_from_filename(
                            path, cb=cb, num_cb=num_cb, policy=grant
                            )
                    else:
                        k.set_contents_from_filename(
                            path, cb=cb, num_cb=num_cb, policy=grant,
                            reduced_redundancy=reduced, encrypt_key=encrypt
                            )
                    
                    # Clean stdout
                    print ''
    
    elif xfer_type == 'download':
        # Perform cloud "download"
        
        if cloud_path:
            cloud_path_key = b.get_key(cloud_path)
        else:
            cloud_path_key = None
        
        if cloud_path_key and not cloud_path_key.name.endswith('/'):
            # Single file download
            key = cloud_path_key
            keypath = key.name.split('/')[-1]
            if not os.path.isdir(path) and not path.endswith(os.sep):
                rename = True
                fullpath = path
            else:
                fullpath = os.path.join(path, keypath)
            
            if key.size == 0:
                if ignore_empty:
                    if not quiet:
                        if rename:
                            print 'Skipping %s -> %s (empty key)' % (
                                keypath, fullpath.split(os.sep)[-1]
                                )
                        else:
                            print 'Skipping %s (empty key)' % (
                                fullpath.split(os.sep)[-1]
                                )
                    copy_file = False
            
            if not os.path.isdir(os.path.split(fullpath)[0]):
                if not quiet:
                    print 'Creating new directory: %s' % (
                        os.path.split(fullpath)[0]
                        )
                if not no_op:
                    os.makedirs(os.path.split(fullpath)[0])
            elif os.path.exists(fullpath):
                if no_overwrite:
                    if not quiet:
                        if rename:
                            print 'Skipping %s -> %s (not overwriting)' % (
                                keypath, fullpath.split(os.sep)[-1]
                                )
                        else:
                            print 'Skipping %s (not overwriting)' % (
                                fullpath.split(os.sep)[-1]
                                )
                    copy_file = False
                elif key.size == os.path.getsize(fullpath) or \
                     key.name.endswith('/'):
                    if not quiet:
                        if rename:
                            print 'Skipping %s -> %s (size matches)' % (
                                keypath.replace('/', os.sep),
                                fullpath.split(os.sep)[-1]
                                )
                        else:
                            print 'Skipping %s (size matches)' % (
                                fullpath.split(os.sep)[-1]
                                )
                    copy_file = False
            
            if copy_file:
                if rename:
                    print '%s -> %s' % (keypath, fullpath.split(os.sep)[-1])
                else:
                    print '%s' % keypath
                
                if not no_op:
                    # Setup callback
                    num_cb = int(key.size ** .25)
                    
                    # Get the file
                    key.get_contents_to_filename(fullpath, cb=cb, num_cb=num_cb)
                    
                    # Clean stdout
                    print ''
        
        else:
            # Possible multi file download
            if not cloud_path_key and cloud_path and \
               not cloud_path.endswith('/'):
                cloud_path += '/'
            
            keys = []
            
            print 'Scanning for keys to transfer...'
            
            for key in b.list(prefix=cloud_path):
                keypath = key.name[len(cloud_path):]
                if cloud_service == 'gs':
                    fullpath = os.path.join(
                        path,
                        keypath.replace('_$folder$', os.sep)
                        )
                else:
                    fullpath = os.path.join(path, keypath.replace('/', os.sep))
                keys.append(fullpath)
                
                if key.size == 0:
                    if ignore_empty:
                        if not quiet:
                            print 'Skipping %s (empty key)' % (
                                fullpath[len(os.path.join(path, '')):]
                                )
                        continue
                
                if not os.path.isdir(os.path.split(fullpath)[0]):
                    if not quiet:
                        print 'Creating new directory: %s' % (
                            os.path.split(fullpath)[0]
                            )
                    if not no_op:
                        os.makedirs(os.path.split(fullpath)[0])
                elif os.path.exists(fullpath):
                    if no_overwrite:
                        if not quiet:
                            print 'Skipping %s (not overwriting)' % (
                                fullpath[len(os.path.join(path, '')):]
                                )
                        continue
                    elif key.size == os.path.getsize(fullpath) or \
                         key.name.endswith('/') or \
                         key.name.endswith('_$folder$'):
                        if not quiet:
                            print 'Skipping %s (size matches)' % (
                                fullpath[len(os.path.join(path, '')):]
                                )
                        continue
                
                if cloud_service == 'gs':
                    print '%s' % (keypath.replace('_$folder$', os.sep))
                else:
                    print '%s' % (keypath.replace('/', os.sep))
                
                if not no_op:
                    if key.name.endswith('/') or key.name.endswith('_$folder$'):
                        # Looks like a directory, so just print the status
                        submit_cb(0, 0)
                    else:
                        # Setup callback
                        num_cb = int(key.size ** .25)
                        
                        # Get the file
                        key.get_contents_to_filename(fullpath, cb=cb,
                                                     num_cb=num_cb)
                    
                    # Clean stdout
                    print ''
            
            # If specified, perform deletes
            if delete:
                for root, dirs, files in os.walk(path):
                    if files:
                        for file in files:
                            filepath = os.path.join(root, file)
                            if filepath not in keys:
                                print 'deleting %s' % (
                                  filepath[len(os.path.join(path, '')):]
                                  )
                                if not no_op:
                                    # Delete the file
                                    os.remove(filepath)
                    elif root != path:
                        dirpath = os.path.join(root, '')
                        if dirpath not in keys:
                            print 'deleting %s' % (
                              dirpath[len(os.path.join(path, '')):]
                              )
                            if not no_op:
                                # Remove the directory
                                os.rmdir(dirpath)
    else:
        # Perform cloud to cloud "sync"
        
        if cloud_path:
            cloud_path_key = b.get_key(cloud_path)
        else:
            cloud_path_key = None
        
        if cloud_path_key and not cloud_path_key.name.endswith('/'):
            # Single file sync
            key = cloud_path_key
            keypath = key.name.split('/')[-1]
            if cloud_dest_path and not cloud_dest_path.endswith('/'):
                rename = True
                fullpath = cloud_dest_path
            else:
                fullpath = cloud_dest_path + keypath
                fullpath = fullpath.lstrip('/')
            
            dest_key = b2.get_key(fullpath)
            
            if key.size == 0:
                if ignore_empty:
                    if not quiet:
                        if rename:
                            print 'Skipping %s -> %s (empty key)' % (
                                keypath.split('/')[-1], fullpath.split('/')[-1]
                                )
                        else:
                            print 'Skipping %s (empty key)' % fullpath
                    copy_file = False
            
            if dest_key:
                # TODO: Check for differing ACL
                if no_overwrite:
                    if not quiet:
                        if rename:
                            print 'Skipping %s -> %s (not overwriting)' % (
                                keypath.split('/')[-1], fullpath.split('/')[-1]
                                )
                        else:
                            print 'Skipping %s (not overwriting)' % fullpath
                    copy_file = False
                elif key.size == dest_key.size:
                    if not quiet:
                        if rename:
                            print 'Skipping %s -> %s (size matches)' % (
                                keypath.split('/')[-1], fullpath.split('/')[-1]
                                )
                        else:
                            print 'Skipping %s (size matches)' % fullpath
                    copy_file = False
            
            if copy_file:
                if rename:
                    sys.stdout.write('%s -> %s...  ' % (
                        keypath.split('/')[-1], fullpath.split('/')[-1])
                        )
                else:
                    sys.stdout.write('%s...  ' % keypath)
                sys.stdout.flush()
                if not no_op:
                    speeds.append((0, time.time()))
                    
                    # Start "spinner" thread
                    ev = threading.Event()
                    t1 = threading.Thread(target=spinner, args=(ev, 0.25))
                    t1.start()
                    
                    try:
                        # Transfer the key
                        key.copy(cloud_dest_bucket, fullpath,
                                 reduced_redundancy=reduced,
                                 preserve_acl=preserve, encrypt_key=encrypt)
                    except Exception, e:
                        raise e
                    finally:
                        # End "spinner" thread
                        ev.set()
                        t1.join()
                    
                    if rename:
                        sys.stdout.write('\r%s -> %s    \n' % (
                            keypath.split('/')[-1], fullpath.split('/')[-1]
                            ))
                    else:
                        sys.stdout.write('\r%s    \n' % keypath)
                    sys.stdout.flush()
                    submit_cb(key.size, key.size)
                else:
                    if rename:
                        sys.stdout.write('\r%s -> %s    ' % (
                            keypath.split('/')[-1], fullpath.split('/')[-1])
                            )
                    else:
                        sys.stdout.write('\r%s    ' % keypath)
                    sys.stdout.flush()
                
                # Clean stdout
                print ''
        
        else:
            # Possible multi file sync
            if not cloud_path_key and cloud_path and \
               not cloud_path.endswith('/'):
                cloud_path += '/'
            if cloud_dest_path and not cloud_dest_path.endswith('/'):
                cloud_dest_path += '/'
            
            keys = []
            
            print 'Scanning for keys to transfer...'
            
            for key in b.list(prefix=cloud_path):
                if key.name == cloud_path:
                    keypath = key.name.split('/')[-2] + '/'
                else:
                    keypath = key.name[len(cloud_path):]
                fullpath = cloud_dest_path + keypath
                fullpath = fullpath.lstrip('/')
                
                keys.append(fullpath)
                dest_key = b2.get_key(fullpath)
                
                if key.size == 0:
                    if ignore_empty:
                        if not quiet:
                            print 'Skipping %s (empty key)' % (
                                fullpath.replace('_$folder$', '/')
                                )
                        continue
                
                if dest_key:
                    # TODO: Check for differing ACL
                    if no_overwrite:
                        if not quiet:
                            print 'Skipping %s (not overwriting)' % (
                                fullpath.replace('_$folder$', '/')
                                )
                        continue
                    elif key.size == dest_key.size:
                        if not quiet:
                            print 'Skipping %s (size matches)' % (
                                fullpath.replace('_$folder$', '/')
                                )
                        continue
                
                sys.stdout.write('%s...  ' % keypath.replace('_$folder$', '/'))
                sys.stdout.flush()
                if not no_op:
                    speeds.append((0, time.time()))
                    
                    # Start "spinner" thread
                    ev = threading.Event()
                    t1 = threading.Thread(target=spinner, args=(ev, 0.25))
                    t1.start()
                    
                    # Transfer the key
                    try:
                        key.copy(cloud_dest_bucket, fullpath,
                                 reduced_redundancy=reduced,
                                 preserve_acl=preserve, encrypt_key=encrypt)
                    except Exception, e:
                        raise e
                    finally:
                        # End "spinner" thread
                        ev.set()
                        t1.join()
                    
                    sys.stdout.write('\r%s    \n' % \
                                     keypath.replace('_$folder$', '/'))
                    sys.stdout.flush()
                    submit_cb(key.size, key.size)
                else:
                    sys.stdout.write('\r%s    ' % \
                                     keypath.replace('_$folder$', '/'))
                    sys.stdout.flush()
                
                # Clean stdout
                print ''
            
            # If specified, perform deletes
            if delete:
                for key in b2.list(prefix=cloud_dest_path):
                    keypath = key.name[len(cloud_dest_path):]
                    
                    if key.name not in keys:
                        print 'deleting %s' % keypath.replace('_$folder$', '/')
                        if not no_op:
                            # Delete the key
                            key.delete()

if __name__ == "__main__":
    main()
