#!/usr/bin/env python -tt
# encoding: utf-8
# PYTHON_ARGCOMPLETE_OK
# ===  IMPORTANT  ====
# NOTE: In order to support non-ASCII file names,
#       your system's locale MUST be set to 'utf-8'
# CAVEAT: DOESN'T seem to work with proxy, the underlying reason being
#         the 'requests' package used for http communication doesn't seem
#         to work properly with proxies, reason unclear.
# NOTE: It seems Baidu doesn't handle MD5 quite right after combining files,
#       so it may return erroneous MD5s. Perform a rapidupload again may fix the problem.
#        That's why I changed default behavior to no-verification.
# NOTE: syncup / upload, syncdown / downdir are partially duplicates
#       the difference: syncup/down compare and perform actions
#       while down/up just proceed to download / upload (but still compare during actions)
#       so roughly the same, except that sync can delete extra files
#
# TODO: Dry run?
# TODO: Use batch functions for better performance
# == NOTE ==
# Proxy is supported by the underlying Requests library,
# you can activate HTTP proxies by setting the HTTP_PROXY and / or
# HTTPS_PROXY environment variables respectively as follows:
#   HTTP_PROXY=http://user:password@domain
#   HTTPS_PROXY=http://user:password@domain
# (More information: http://docs.python-requests.org/en/master/user/advanced/#proxies)
# Though from my experience, it seems that some proxy servers may not be supported properly.

# from __future__ imports must occur at the beginning of the file
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division

import os
import sys
import io
import signal
import time
import shutil
import tempfile
import posixpath
import json
import hashlib
#import base64
import re
import pprint
import socket
import subprocess
import shlex
import atexit
from collections import deque
#from collections import OrderedDict
from functools import partial
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
from copy import deepcopy

# unify Python 2 and 3
if sys.version_info[0] == 2:
	import urllib as ulp
	import cPickle as pickle
	pickleload = pickle.load
elif sys.version_info[0] == 3:
	import urllib.parse as ulp
	import pickle
	unicode = str
	basestring = str
	long = int
	raw_input = input
	pickleload = partial(pickle.load, encoding="bytes")

from . import const
from . import gvar
from . import printer_console
from . import cached as cachedm
from . import util
from . import printer
from .cached import (cached, stringifypickle, md5, crc32, slice_md5)
from .struct import PathDictTree
from .util import (
	iswindows,
	nop,
	perr, pwarn, pinfo, pdbg,
	jsondump, jsonload, formatex, rb,
	joinpath, get_pcs_path, print_pcs_list, str2bool, str2int,
	human_size, interpret_size, ls_time, ls_type,
	makedir, removedir, movefile, removefile, getfilesize,
	comp_semver,
	MyPrettyPrinter)
from .chkreq import (check_requirements, CheckResult)
from .requester import RequestsRequester

pr = printer_console.pr
prcolor = printer_console.prcolor
ask = printer_console.ask
pprgr = printer_console.pprgr

# there was a WantWriteError uncaught exception for Urllib3:
# https://github.com/shazow/urllib3/pull/412
# it was fixed here:
# https://github.com/shazow/urllib3/pull/413
# commit:
# https://github.com/shazow/urllib3/commit/a89dda000ed144efeb6be4e0b417c0465622fe3f
# and this was included in this commit in the Requests library
# https://github.com/kennethreitz/requests/commit/7aa6c62d6d917e11f81b166d1d6c9e60340783ac
# which was included in version 2.5.0 or above
# so minimum 2.5.0 is required
import requests
try:
	from requests.packages.urllib3.exceptions import ReadTimeoutError
except:
	try:
		from urllib3.exceptions import ReadTimeoutError
		ReadTimeoutError
	except:
		perr("Something seems wrong with the urllib3 installation.\nQuitting")
		sys.exit(const.EFatal)

from requests_toolbelt import multipart

# http://stackoverflow.com/a/27320254/404271
try:
	import multiprocess as mp
	from multiprocess import Pool
except ImportError:
	mp = None
	Pool = None
	perr("'multiprocess' library is not available, no parallel dl/ul.")

mpsemaphore = mp.Semaphore(1) if mp else None
cached.semaphore = mpsemaphore

if Pool != None:
	if sys.version_info[0] == 2:
		# enable with statement
		from contextlib import contextmanager
		@contextmanager
		def UPool(*args, **kwargs):
			yield Pool(*args, **kwargs)
	elif sys.version_info[0] == 3:
		UPool = Pool

pcsurl = const.PcsUrl
cpcsurl = const.CPcsUrl
dpcsurl = const.DPcsUrl

# This crap is here to avoid circular imports
# What a fantastic packing architecture Python has!
def makemppr(pr):
	def mppr(msg, *args, **kwargs):
		return pr(mp.current_process().name + ': ' + msg, *args, **kwargs)
	return mppr

def makemppprgr(pprgr):
	def mppprgr(finish, total, start_time = None, existing = 0,
		prefix = '', suffix = '', seg = 20):
		prefix = mp.current_process().name + ': ' + str(prefix)
		pprgr(finish, total, start_time, existing, prefix, suffix, seg)
	return mppprgr

def set_mp_print():
	global pr
	global prcolor
	global ask
	global pprgr
	opr = pr
	oprcolor = prcolor
	oask = ask
	opprgr = pprgr
	def restoremp():
		global pr
		global prcolor
		global ask
		global pprgr
		pr = cachedm.pr = util.pr = opr
		prcolor = util.prcolor = printer.prcolor = oprcolor
		ask = util.ask = oask
		pprgr = util.pprgr = opprgr

	pr = cachedm.pr = util.pr = makemppr(opr)
	prcolor = util.prcolor = printer.prcolor = makemppr(oprcolor)
	ask = util.ask = makemppr(oask)
	pprgr = util.pprgr = makemppprgr(opprgr)

	return restoremp

class ByPy(object):
	'''The main class of the bypy program'''
	# TODO: Apply to configdir instead of ~/.bypy
	@staticmethod
	def migratesettings():
		result = const.ENoError

		if os.path.exists(const.OldByPyCertsPath):
			removefile(const.OldByPyCertsPath)

		filesToMove = [
			[const.OldTokenFilePath, const.TokenFilePath],
			[const.OldPicklePath, const.PicklePath]
		]

		result = makedir(const.ConfigDir, 0o700) and result # make it secretive
		# this directory must exist
		if result != const.ENoError:
			perr("Fail to create config directory '{}'".format(const.ConfigDir))
			return result

		for tomove in filesToMove:
			oldfile = tomove[0]
			newfile = tomove[1]
			if os.path.exists(oldfile):
				dst = newfile
				if os.path.exists(newfile):
					dst = dst + '.old'
				result = movefile(oldfile, dst) and result

		# we move to JSON for hash caching for better portability
		# http://www.benfrederickson.com/dont-pickle-your-data/
		# https://kovshenin.com/2010/pickle-vs-json-which-is-faster/
		# JSON even outpeforms Pickle and definitely much more portable
		# DON'T bother with pickle.
		if os.path.exists(const.PicklePath):
			oldcache = {}
			try:
				with io.open(const.PicklePath, 'rb') as f:
					oldcache = pickleload(f)
				stringifypickle(oldcache)
				cached.loadcache(oldcache)
				cached.savecache(force_saving=True)
				pinfo("Contents of Pickle (old format hash cache) '{}' "
				"has been merged to '{}'".format(const.PicklePath, const.HashCachePath))
				mergedfile = const.PicklePath + '.merged'
				ok = movefile(const.PicklePath, mergedfile)
				if ok == const.ENoError:
					pinfo("Pickle (old format hash cache) '{}' "
					"has been renamed to '{}".format(const.PicklePath, mergedfile))
				else:
					perr("Failed to move Pickle (old format hash cache) '{}' to '{}'".format(const.PicklePath, mergedfile))
			except (
				pickle.PickleError,
				# the following is for dealing with corrupted cache file
				EOFError, TypeError, ValueError):
				invalidfile = const.PicklePath + '.invalid'
				ok = movefile(const.PicklePath, invalidfile)
				perr("{} invalid Pickle (old format hash cache) file '{}' to '{}'".format(
					"Moved" if ok == const.ENoError else "Failed to move",
					const.PicklePath, invalidfile))

		return result

	def setpcsurl(self, mirror):
		global pcsurl
		global cpcsurl
		global dpcsurl
		mirror = 'https://' + mirror + const.RestApiPath
		pcsurl = mirror
		cpcsurl = mirror
		dpcsurl = mirror
		# TODO: the SSL error may have gone, and the following is not needed
#		if mirror != const.PcsDomain:
#			# using a mirror, which has name mismatch SSL error, so need to disable SSL check
#			pwarn("Mirror '{}' used instead of the default PCS server url '{}', ".format(pcsurl, const.PcsUrl) +  \
#				  "we have to disable the SSL cert check in this case.")
#			self._checkssl = False

	def savesetting(self):
		try:
			jsondump(self._setting, self._settingpath, mpsemaphore)
		except Exception as ex:
			perr("Failed to save settings.\n{}".format(formatex(ex)))
			# complaining is enough, no need more actions as this is non-critical

	# def quit(self, retcode = const.ENoError):
	def cleanup(self):
		# saving is the most important
		# we save, but don't clean, why?
		# think about unmount path, moved files,
		# once we discard the information, they are gone.
		# so unless the user specifically request a clean,
		# we don't act too smart.
		#cached.cleancache()
		cached.savecache()
		self.savesetting()
		# if we flush() on Ctrl-C, we get
		# IOError: [Errno 32] Broken pipe
		sys.stdout.flush()
		# sys.exit(retcode)

	# TODO: this constructor is getting fat ...
	def __init__(self,
		slice_size = const.DefaultSliceSize,
		dl_chunk_size = const.DefaultDlChunkSize,
		verify = True,
		retry = 5, timeout = const.DefaultTimeOutInSeconds,
		quit_when_fail = False,
		resumedownload = True,
		extraupdate = lambda: (),
		incregex = '',
		ondup = '',
		followlink = True,
		checkssl = True,
		cacerts = None,
		rapiduploadonly = False,
		mirror = '',
		selectmirror = False,
		resumedl_revertcount = const.DefaultResumeDlRevertCount,
		deletesource = False,
		verbose = 0, debug = False,
		configdir = const.ConfigDir,
		requester = RequestsRequester,
		apikey = const.ApiKey,
		downloader = "",
		downloader_args = "",
		processes = const.DefaultProcessCount,
		secretkey = const.SecretKey):
		super(ByPy, self).__init__()
		self.jsonq = deque(maxlen = 64)

		# these two variables are without leading double underscore "__" as to export the as public,
		# so if any code using this class can check the current verbose / debug level
		cached.verbose = self.verbose = verbose
		cached.debug = self.debug = debug

		self._load_auth_server_list()

		if not cached.usecache:
			pinfo("Forced hash recalculation, hash cache won't be used")

		# handle backward compatibility, a.k.a. history debt
		sr = ByPy.migratesettings()
		if sr != const.ENoError:
			# bail out
			perr("Failed to migrate old settings.")
			sys.exit(const.EMigrationFailed)

		self._configdir = configdir.rstrip("/\\ ")
		makedir(self._configdir)
		# os.path.join() may not handle unicode well on Python 2.7
		self._progresspath = configdir + os.sep + const.ProgressFileName
		self._tokenpath = configdir + os.sep + const.TokenFileName
		self._settingpath = configdir + os.sep + const.SettingFileName
		self._setting = {}

		self._downloader = downloader.lower().strip()
		if downloader_args:
			self._downloader_args = downloader_args
		else:
			if downloader in const.DownloaderDefaultArgs:
				self._downloader_args = const.DownloaderDefaultArgs[downloader]
			else:
				self._downloader_args = ''

		if os.path.exists(self._settingpath):
			try:
				self._setting = jsonload(self._settingpath)
			except Exception as ex:
				perr("Error loading settings, using default settings.{}".format(formatex(ex)))
		self._hashcachepath = configdir + os.sep + const.HashCacheFileName
		cached.hashcachepath = self._hashcachepath
		self._certspath = os.path.join(os.path.dirname(__file__), const.ByPyCertsFileName)

		self._requester = requester
		self._apikey = apikey
		self._secretkey = secretkey
		self._use_server_auth = not secretkey

		self._slice_size = slice_size
		self._dl_chunk_size = dl_chunk_size
		self._verify = verify
		self._retry = retry
		self._quit_when_fail = quit_when_fail
		self._timeout = timeout
		self._resumedownload = resumedownload
		self._extraupdate = extraupdate
		self._incregex = incregex
		self._incregmo = re.compile(incregex)
		if ondup and len(ondup) > 0:
			self._ondup = ondup[0].upper()
		else:
			self._ondup = 'O' # O - Overwrite* S - Skip P - Prompt
		self._followlink = followlink
		self._rapiduploadonly = rapiduploadonly
		self._resumedl_revertcount = resumedl_revertcount
		self._deletesource = deletesource
		if deletesource:
			self.pd("Forcing verification since we will delete source for successful transfers.")
			self._verify = True
		self.processes = processes

		# in case of abortions, exceptions, etc
		atexit.register(self.cleanup)

		#TODO: SSL verification causes some much trouble for different Python version
		# I give up and disable it for good, or for bad
		checkssl = False
		# set SSL Check flag before mirror configuration, which may change it
		self._checkssl = checkssl
		# --mirror takes precedence over --select-fastest-mirror
		mirror = mirror.lower()
		if len(mirror) > 0 and mirror != const.PcsDomain:
			self.setpcsurl(mirror)
		elif selectmirror:
			self._select_fastest_mirror()
		# else: already set in global

		if self._checkssl:
			# sort of undocumented by requests
			# http://stackoverflow.com/questions/10667960/python-requests-throwing-up-sslerror
			if cacerts is not None:
				if os.path.isfile(cacerts):
					self._certspath = cacerts
				else:
					perr("Invalid CA Bundle '{}' specified")

			# falling through here means no customized CA Certs specified
			if self._checkssl is True:
				# use our own CA Bundle if possible
				if os.path.isfile(self._certspath):
					self._checkssl = self._certspath
				else:
					# Well, disable cert verification
					pwarn(
"** SSL Certificate Verification has been disabled **\n\n"
"If you are confident that your CA Bundle can verify "
"Baidu PCS's certs, you can run the prog with the '" + const.CaCertsOption + \
" <your ca cert path>' argument to enable SSL cert verification.\n\n"
"However, most of the time, you can ignore this warning, "
"you are not going to send sensitive data to the cloud plainly right?")
					self._checkssl = False
		if not checkssl:
			requester.disable_warnings(self.debug)

		cached.loadcache()
		requester.set_logging_level(debug)
		# useful info for debugging
		if debug > 0:
			pr("----")
			pr("Verbose level = {}".format(verbose))
			pr("Debug level = {}".format(debug))
			# these informations are useful for debugging
			pr("Config directory: '{}'".format(self._configdir))
			pr("Token file: '{}'".format(self._tokenpath))
			pr("Hash Cache file: '{}'".format(self._hashcachepath))
			pr("App root path at Baidu Yun '{}'".format(const.AppPcsPath))
			pr("sys.stdin.encoding = {}".format(sys.stdin.encoding))
			pr("sys.stdout.encoding = {}".format(sys.stdout.encoding))
			pr("sys.stderr.encoding = {}".format(sys.stderr.encoding))
			pr("----\n")

		# the prophet said: thou shalt initialize
		self._existing_size = 0
		self._json = {}
		self._access_token = ''
		self._remote_json = {}
		self._slice_md5s = []
		self._cookies = {}
		# TODO: whether this works is still to be tried out
		self._isrev = False
		self._rapiduploaded = False

		# store the response object, mainly for testing.
		self.response = object()
		# store function-specific result data
		self.result = {}

		if not self._load_local_json():
			# no need to call _load_local_json() again as _auth() will load the json & acess token.
			result = self._auth()
			if result != const.ENoError:
				perr("Program authorization FAILED.\n"
					"You need to authorize this program before using any PCS functions.\n"
					"Quitting...\n")
				sys.exit(result)

		for proxy in ['HTTP_PROXY', 'HTTPS_PROXY']:
			if proxy in os.environ:
				pr("{} used: {}".format(proxy, os.environ[proxy]))

		# update check
		check_update = False
		nowsec = int(time.time())
		if const.SettingKey_LastUpdateCheckTime not in self._setting:
			check_update = True
		else:
			lastcheck = self._setting[const.SettingKey_LastUpdateCheckTime]
			if nowsec - lastcheck > 7 * 24 * 60 * 60: # check every 7 days
				check_update = True

		if check_update:
			try:
				r = requests.get('https://raw.githubusercontent.com/houtianze/bypy/master/update/update.json')
				if r.status_code == 200:
					try:
						j = r.json()
						min_ver_key = 'minimumRequiredVersion'
						if min_ver_key in j:
							minver = j[min_ver_key]
							if comp_semver(const.__version__, minver) < 0:
								perr("Your current version ({}) is too low, "
									"minimum required version is {}.\n"
									"Please run 'pip install -U bypy' to update and try again.".format(
										const.__version__, minver))
								sys.exit(const.EUpdateNeeded)
							else:
								self._setting[const.SettingKey_LastUpdateCheckTime] = nowsec
								self.savesetting()

						recommended_ver_key = 'recommendedVersion'
						if recommended_ver_key in j:
							recver = j[recommended_ver_key]
							if comp_semver(const.__version__, recver) < 0:
								pr("Your current version ({}) is low, "
									"It's recommended to update to version {}.\n"
									"Please run 'pip install -U bypy' to update.".format(
										const.__version__, recver))
					except ValueError:
						self.pd("Invalid response for update check, skipping.")
				else:
					self.pd("HTTP Status {} while checking update, skipping.".format(r.status_code))
			except:
				self.pd("Error occurred while checking update, skipping.")

	def pv(self, msg, **kwargs):
		if self.verbose:
			pr(msg)

	def pd(self, msg, level = 1, **kwargs):
		if self.debug and self.debug >= level:
			pdbg(msg, kwargs)

	def shalloverwrite(self, prompt):
		if self._ondup == 'S':
			return False
		elif self._ondup == 'P':
			ans = ask(prompt, False).upper()
			if not ans.startswith('Y'):
				return False

		return True

	def _warn_multi_processes(self, process = "dl / ul"):
		if process:
			pwarn("Running multiple ({}) processes for {}, I don't know if things will go awry...".format(
				self.processes, process))

	def _check_prompt_multiprocess(self):
		if self._ondup == 'P' and Pool and self.processes != 1:
			errmsg = "Can't do parallel processing if you want to prompt the user for overwrite confirmation"
			perr(errmsg)
			return False
		return True

	def _filter_multi_results(self, results):
		# http://www.diveintopython3.net/porting-code-to-python-3-with-2to3.html#filter
		errors = list(filter(lambda x: x != const.ENoError, results))
		return const.ENoError if len(errors) == 0 else errors[-1]

	def _multi_process(self, worker, iterator, process = "dl / ul"):
		if not self._check_prompt_multiprocess():
			return const.EArgument
		self._warn_multi_processes(process)
		# patch the printer
		restoremp = set_mp_print()
		with UPool(self.processes) as pool:
			try:
				# http://stackoverflow.com/a/35134329/404271
				# On Python 2.x, Ctrl-C won't work if we use pool.map(), wait() or get()
				ar = pool.map_async(worker, iterator)
				#results = ar.get(const.TenYearInSeconds)
				results = ar.get(const.FortyNineDaysInSeconds)
				restoremp()
				return self._filter_multi_results(results)
			except pickle.PicklingError as pe:
				errmsg = ("Exception:\n{}\n"
						"--------------------------------\n"
						"Error: Your Python 'multiprocess' library is probably "
						"not properly installed (missing C extensions). "
						"You need to install a C compiler and Python headers before "
						"installing the Python 'multiprocess' library. "
						"(All these hassles could have been saved if Python's builtin "
						"'multiprocessing' works properly, sigh.)\n"
						"Fix for debian derivatives:\n"
						"- Install gcc: # apt-get install gcc\n"
						"- Install python-dev: # apt-get install python-dev\n"
						"- Reinstall Python 'multiprocess' library:\n"
						"  # pip uninstall -y multiprocess\n"
						"  # pip install -v multiprocess\n"
						"- If there's no errors/warnings in the above actions, "
						"then this error should be gone when you run 'bypy' with '{}' again.\n"
						).format(formatex(pe), const.MultiprocessOption)
				perr(errmsg)
				sys.exit(const.EFatal)

	def _print_error_json(self, r):
		try:
			dj = r.json()
			if 'error_code' in dj and 'error_msg' in dj:
				ec = dj['error_code']
				et = dj['error_msg']
				msg = ''
				if ec == const.IEMD5NotFound:
					pf = pinfo
					msg = et
				else:
					pf = perr
					msg = "Error JSON returned:{}\nError code: {}\nError Description: {}".format(dj, ec, et)
				pf(msg)
		except Exception as ex:
			perr('Error parsing JSON Error Code from:\n{}\n{}'.format(rb(r.text), formatex(ex)))

	def _dump_exception(self, ex, url, pars, r, act):
		if self.debug or self.verbose:
			perr("Error accessing '{}'".format(url))
			if self.debug:
				perr(formatex(ex))
			perr("Function: {}".format(act.__name__))
			perr("Website parameters: {}".format(pars))
			if r != None:
				# just playing it safe
				if hasattr(r, 'url'):
					perr("Full URL: {}".format(r.url))
				if hasattr(r, 'status_code') and hasattr(r, 'text'):
					perr("HTTP Response Status Code: {}".format(r.status_code))
					if (r.status_code != 200 and r.status_code != 206) \
						or (not ('method' in pars and pars['method'] == 'download') \
							and url.find('method=download') == -1 \
							and url.find('baidupcs.com/file/') == -1):
						self._print_error_json(r)
						perr("Website returned: {}".format(rb(r.text)))

	# child class override this to to customize error handling
	def _handle_more_response_error(self, r, sc, ec, act, actargs):
		return const.ERequestFailed

	# Workaround for Baidu returning wrong HTTP status code
	# Who in one's right mind would send error with an HTTP 200 status?
	# <D> HTTP Status Code: 200
	# <D> 200 OK, processing action
	# <D> Response: {'error_code': 111, 'error_msg': 'Access token expired'}
	def _is_error_response(self, r):
		# don't attempt on large contents
		isJson = False
		ContentTypeKey = 'Content-Type'
		contentType = r.headers.get(ContentTypeKey)
		if (contentType and 'json' in contentType):
			isJson = True

		if len(r.content) > 10 * 1024 or not isJson:
			return False
		try:
			j = r.json()
			if 'error_code' in j and j['error_code'] != 0:
				return True
		except ValueError:
			return False
		return False

	def _get_json_errorcode(self, r, act, defaultec = const.ERequestFailed):
		try:
			j = r.json()
			self.pd("Website returned JSON: {}".format(j))
			if 'error_code' in j:
				return j['error_code']
			else:
				return defaultec
		except ValueError:
			if hasattr(r, 'text'):
				self.pd("Website Response: {}".format(rb(r.text)))
			if act == self._cdl_act:
				return const.IETaskNotFound
			return defaultec

	def _request_work_die(self, ex, url, pars, r, act):
		result = const.EFatal
		self._dump_exception(ex, url, pars, r, act)
		perr("Fatal Exception, no way to continue.\nQuitting...\n")
		perr("If the error is reproducible, run the program with `-dv` arguments again to get more info.\n")
		sys.exit(result)
		# we eat the exception, and use return code as the only
		# error notification method, we don't want to mix them two
		#raise # must notify the caller about the failure

	def _request_work(self, url, pars, act, method, actargs = None, addtoken = True, dumpex = True, **kwargs):
		result = const.ENoError
		r = None

		self._extraupdate()
		parsnew = pars.copy()
		if addtoken:
			parsnew['access_token'] = self._access_token

		try:
			self.pd(method + ' ' + url)
			self.pd("actargs: {}".format(actargs))
			self.pd("Params: {}".format(pars))

			r = self._requester.request(method, url, params = parsnew, timeout = self._timeout, verify = self._checkssl, **kwargs)
			self.response = r
			sc = r.status_code
			self.pd("Full URL: {}".format(r.url))
			self.pd("HTTP Status Code: {}".format(sc))
			# BUGFIX: DON'T do this, if we are downloading a big file,
			# the program will eat A LOT of memeory and potentially hang / get killed
			#self.pd("Request Headers: {}".format(pprint.pformat(r.request.headers)), 2)
			#self.pd("Response Header: {}".format(pprint.pformat(r.headers)), 2)
			#self.pd("Response: {}".format(rb(r.text)), 3)
			# 206 Partial Content
			if (sc == requests.codes.ok or sc == 206) and not self._is_error_response(r):
				if sc == requests.codes.ok:
					# #162 https://github.com/houtianze/bypy/pull/162
					# handle response like this:  {"error_code":0,"error_msg":"no error","request_id":70768340515255385}
					if not ('method' in pars and pars['method'] == 'download'):
						try:
							j = r.json()
							if 'error_code' in j and j['error_code'] == 0 and 'error_msg' in j and j['error_msg'] == 'no error':
								self.pd("Unexpected response: {}".format(j))
								return const.ERequestFailed
						except Exception as ex:
							perr(formatex(ex))
							# TODO: Shall i return this?
							return const.ERequestFailed

					self.pd("200 OK, processing action")
				else:
					self.pd("206 Partial Content (this is OK), processing action")
				result = act(r, actargs)
				if result == const.ENoError:
					self.pd("Request OK")
			else:
				ec = self._get_json_errorcode(r, act)
				#   6 (sc: 403): No permission to access user data
				# 110 (sc: 401): Access token invalid or no longer valid
				# 111 (sc: 401): Access token expired
				if ec == 111 or ec == 110 or ec == 6 or ec == const.IEAppIDIsEmpty: # and sc == 401:
					self.pd("ec = {}".format(ec))
					self.pd("Need to refresh token, refreshing")
					if const.ENoError == self._refresh_token(): # refresh the token and re-request
						# TODO: avoid infinite recursive loops
						# TODO: properly pass retry
						result = self._request(url, pars, act, method, actargs, True, addtoken, dumpex, **kwargs)
					else:
						result = const.EFatal
						perr("FATAL: Token refreshing failed, can't continue.\nQuitting...\n")
						sys.exit(result)
				# File md5 not found, you should use upload API to upload the whole file.
				elif ec == const.IEMD5NotFound: # and sc == 404:
					self.pd("MD5 not found, rapidupload failed")
					result = ec
				# superfile create failed
				elif ec == const.IESuperfileCreationFailed \
					or ec == const.IEBlockMissInSuperFile2:
					self.pd("Failed to combine files from MD5 slices")
					result = ec
				elif ec == const.IEFileAlreadyExists: # ec == 31061, sc == 400, file already exists
					# explanations:
					# this error occurs when we try to create a directory at Baidu PCS when it already exists,
					# this almost only happens when we upload using multi-process (double/multiple-directory creation).
					# this error is innocuous, we can safely ignore it
					# (actually, we don't need to do anything further and should just continue as usual).
					# so to avoid confusions to the user and potential logic errors (more importantly) in the future,
					# we just "lie" and force it to "no error".
					# though this is not 100% accurate technically, i feel it's accurate enough for the users
					self.pd("Faking error_code {} to {}, this is safe to ignore.".format(
						const.IEFileAlreadyExists, const.ENoError))
					result = const.ENoError
				# errors that make retrying meaningless
				elif (
					ec == 31062 or # sc == 400 file name is invalid
					ec == 31063 or # sc == 400 file parent path does not exist
					ec == 31064 or # sc == 403 file is not authorized
					ec == 31065 or # sc == 400 directory is full
					ec == 31066 or # sc == 403 (indeed 404) file does not exist
					ec == const.IETaskNotFound or # 36016 or # sc == 404 Task was not found
					ec == const.IEParameterError or # sc == 400 param error
					# the following was found by xslidian, but i have never ecountered before
					ec == 31390):  # sc == 404 # {"error_code":31390,"error_msg":"Illegal File"} # r.url.find('http://bcscdn.baidu.com/bcs-cdn/wenxintishi') == 0
					result = ec
					# TODO: Move this out to cdl_cancel() ?
					#if ec == const.IETaskNotFound:
					#	pr(r.json())
					if dumpex:
						self._dump_exception(None, url, pars, r, act)
				else:
					# gate for child classes to customize behaviors
					# the function should return ERequestFailed if it doesn't handle the case
					result = self._handle_more_response_error(r, sc, ec, act, actargs)
					if result == const.ERequestFailed and dumpex:
						self._dump_exception(None, url, pars, r, act)
		except (requests.exceptions.RequestException,
				socket.error,
				ReadTimeoutError) as ex:
			# If certificate check failed, no need to continue
			# but prompt the user for work-around and quit
			# why so kludge? because requests' SSLError doesn't set
			# the errno and strerror due to using **kwargs,
			# so we are forced to use string matching
			if isinstance(ex, requests.exceptions.SSLError) \
				and re.match(r'^\[Errno 1\].*error:14090086.*:certificate verify failed$', str(ex), re.I):
				# [Errno 1] _ssl.c:504: error:14090086:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed
				result = const.EFatal
				self._dump_exception(ex, url, pars, r, act)
				perr("\n\n== Baidu's Certificate Verification Failure ==\n"
				"We couldn't verify Baidu's SSL Certificate.\n"
				"It's most likely that the system doesn't have "
				"the corresponding CA certificate installed.\n"
				"There are two ways of solving this:\n"
				"Either) Run this prog with the '" + const.CaCertsOption + \
				" <path to " + const.ByPyCertsFileName + "> argument "
				"(" + const.ByPyCertsFileName + " comes along with this prog). "
				"This is the secure way. "
				"However, it won't work after 2020-02-08 when "
				"the certificat expires.\n"
				"Or) Run this prog with the '" + const.DisableSslCheckOption + \
				"' argument. This supresses the CA cert check "
				"and always works.\n")
				# sys.exit(result)
				raise

			# why so kludge? because requests' SSLError doesn't set
			# the errno and strerror due to using **kwargs,
			# so we are forced to use string matching
			if isinstance(ex, requests.exceptions.SSLError) \
				and re.match(r'^\[Errno 1\].*error:14090086.*:certificate verify failed$', str(ex), re.I):
				# [Errno 1] _ssl.c:504: error:14090086:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed
				perr("\n*** We probably don't have Baidu's CA Certificate ***\n" \
				"This in fact doesn't matter most of the time.\n\n" \
				"However, if you are _really_ concern about it, you can:\n" \
				"Either) Run this prog with the '" + const.CaCertsOption + \
				" <path to bypy.cacerts.pem>' " \
				"argument. This is the secure way.\n" \
				"Or) Run this prog with the '" + const.DisableSslCheckOption + \
				"' argument. This suppresses the CA cert check.\n")

			result = const.ERequestFailed
			if dumpex:
				self._dump_exception(ex, url, pars, r, act)

		# TODO: put this check into the specific funcitons?
		except ValueError as ex:
			if ex.message == 'No JSON object could be decoded':
				result = const.ERequestFailed
				if dumpex:
					self._dump_exception(ex, url, pars, r, act)
			else:
				result = const.EFatal
				# self._request_work_die(ex, url, pars, r, act)
				raise

		except Exception as ex:
			# OpenSSL SysCallError
			if ex.args == (10054, 'WSAECONNRESET') \
			or ex.args == (10053, 'WSAECONNABORTED') \
			or ex.args == (104, 'ECONNRESET') \
			or ex.args == (110, 'ETIMEDOUT') \
			or ex.args == (32, 'EPIPE'):
				result = const.ERequestFailed
				if dumpex:
					self._dump_exception(ex, url, pars, r, act)
			else:
				result = const.EFatal
				# self._request_work_die(ex, url, pars, r, act)
				raise

		return result

	def _request(self, url, pars, act, method, actargs = None, retry = True, addtoken = True, dumpex = True, **kwargs):
		tries = 1
		if retry:
			tries = self._retry

		result = const.ERequestFailed

		# Change the User-Agent to avoid server fuss
		kwnew = kwargs.copy()
		if 'headers' not in kwnew:
			kwnew['headers'] = { 'User-Agent': const.UserAgent }

		# Now, allow User-Agent to be set in the caller, instead of always using the default UserAgent value.
		if 'User-Agent' not in kwnew['headers']:
			kwnew['headers']['User-Agent'] = const.UserAgent

		i = 0
		while True:
			result = self._request_work(url, pars, act, method, actargs, addtoken, dumpex, **kwnew)
			i += 1
			# only ERequestFailed needs retry, other error still directly return
			if result == const.ERequestFailed:
				if i < tries:
					# algo changed: delay more after each failure
					delay = const.RetryDelayInSec * i
					perr("Waiting {} seconds before retrying...".format(delay))
					time.sleep(delay)
					perr("Request Try #{} / {}".format(i + 1, tries))
				else:
					result = const.EMaxRetry
					perr("Maximum number ({}) of tries failed.".format(tries))
					if self._quit_when_fail:
						sys.exit(const.EMaxRetry)
					break
			else:
				break

		return result

	def _get(self, url, pars, act, actargs = None, retry = True, addtoken = True, dumpex = True, **kwargs):
		return self._request(url, pars, act, 'GET', actargs, retry, addtoken, dumpex, **kwargs)

	def _post(self, url, pars, act, actargs = None, retry = True, addtoken = True, dumpex = True, **kwargs):
		return self._request(url, pars, act, 'POST', actargs, retry, addtoken, dumpex, **kwargs)

	# direction: True - upload, False - download
	def _shallinclude(self, lpath, rpath, direction):
		arrow = '==>' if direction else '<=='
		checkpath = lpath if direction else rpath
		# TODO: bad practice, see os.access() document for more info
		if direction: # upload
			if not os.path.exists(lpath):
				perr("'{}' {} '{}' skipped since local path no longer exists".format(
					lpath, arrow, rpath))
				return False
		else: # download
			if os.path.exists(lpath) and (not os.access(lpath, os.R_OK)):
				perr("'{}' {} '{}' skipped due to permission".format(
					lpath, arrow, rpath))
				return False

		if '\\' in os.path.basename(checkpath):
			perr("'{}' {} '{}' skipped due to problemic '\\' in the path".format(
				lpath, arrow, rpath))
			return False

		include = (not self._incregex) or self._incregmo.match(checkpath)
		if not include:
			self.pv("'{}' {} '{}' skipped as it's not included in the regex pattern".format(
				lpath, arrow, rpath))

		return include

	ListFormatDict = {
		'$t' : (lambda json: ls_type(json['isdir'])),
		'$f' : (lambda json: json['path'].split('/')[-1]),
		'$c' : (lambda json: ls_time(json['ctime'])),
		'$m' : (lambda json: ls_time(json['mtime'])),
		'$d' : (lambda json: str(json['md5'] if 'md5' in json else '')),
		'$s' : (lambda json: str(json['size'])),
		'$i' : (lambda json: str(json['fs_id'])),
		'$b' : (lambda json: str(json['block_list'] if 'block_list' in json else '')),
		'$u' : (lambda json: 'HasSubDir' if 'ifhassubdir' in json and json['ifhassubdir'] else 'NoSubDir'),
	}

	def _replace_list_format(self, fmt, j):
		output = fmt
		for k, v in ByPy.ListFormatDict.items():
			output = output.replace(k, v(j))
		self.file_list.append(output)
		return output

	def _load_local_json(self):
		try:
			self._json = jsonload(self._tokenpath)
			self._access_token = self._json['access_token']
			self.pd("Token loaded:")
			self.pd(self._json)
			return True
		except Exception as ex:
			self.pd("Error while loading baidu pcs token.\n{}".format(formatex(ex)))
			return False

	def _store_json_only(self, j):
		self._json = j
		self._access_token = self._json['access_token']
		self.pd("access token: " + self._access_token)
		self.pd("Authorize JSON:")
		self.pd(self._json)
		tokenmode = 0o600
		try:
			jsondump(self._json, self._tokenpath, mpsemaphore)
			os.chmod(self._tokenpath, tokenmode)
			return const.ENoError
		except Exception as ex:
			perr("Exception occured while trying to store access token:\n{}".format(
				formatex(ex)))
			return const.EFileWrite

	def _prompt_clean(self):
		pinfo('-' * 64)
		pinfo("""This is most likely caused by authorization errors.
Possible causes:
 - You didn't run this program for a long time (more than a month).
 - You changed your Baidu password after authorizing this program.
 - You didn't give this program the 'netdisk' access while authorizing.
 - ...
Possible fixes:
 1. Remove the authorization token by running with the parameter '{}', and then re-run this program.
 2. If (1) still doesn't solve the problem, you may have to go to:
    https://passport.baidu.com/accountbind
    and remove the authorization of this program, and then re-run this program.""".format(const.CleanOptionShort))
		return const.EInvalidJson

	def _store_json(self, r):
		j = {}
		try:
			j = r.json()
		except Exception as ex:
			perr("Failed to decode JSON:\n{}".format(formatex(ex)))
			perr("Error response:\n{}".format(r.text))
			return self._prompt_clean()

		if 'access_token' not in j:
			perr("Invalid token JSON:\n{}".format(j))
			return const.EInvalidJson
		return self._store_json_only(j)

	def _server_auth_act(self, r, args):
		return self._store_json(r)

	def _local_auth_act(self, r, args):
		return self._store_json(r)

	def _repr_timeout(self):
		return self._timeout if self._timeout else 'infinite'

	def _load_auth_server_list(self):
		# https://stackoverflow.com/a/58941536/404271
		import pkgutil
		j = json.loads(pkgutil.get_data(__name__, 'res/auth.json'))
		self.pd('Auth servers loaded: {}'.format(j))
		self.AuthServerList = j['AuthServerList']
		self.RefreshServerList = j['RefreshServerList']

	def _update_auth_server_list(self):
		try:
			r = requests.get('https://raw.githubusercontent.com/houtianze/bypy/master/bypy/res/auth.json')
			if r.status_code == 200:
				try:
					j = r.json()
					self.pd('Auth servers updated: {}'.format(j))
					self.AuthServerList = j['AuthServerList']
					self.RefreshServerList = j['RefreshServerList']
				except ValueError:
					self.pd("Invalid response for auth servers update, skipping.")
			else:
				self.pd("HTTP Status {} while updating auth servers, skipping.".format(r.status_code))
		except:
			self.pd("Error occurred while updating auth servers, skipping.")

	def _auth(self):
		params = {
			'client_id' : self._apikey,
			'response_type' : 'code',
			'redirect_uri' : 'oob',
			'scope' : 'basic netdisk' }
		pars = ulp.urlencode(params)
		msg = 'Please visit:\n{}\nAnd authorize this app'.format(const.ServerAuthUrl + '?' + pars) + \
			'\nPaste the Authorization Code here within 10 minutes.'
		auth_code = ''
		while True:
			auth_code = ask(msg).strip()
			if auth_code and len(auth_code) >= 16: # 16 is just random lower bound
				break
		self.pd("auth_code: {}".format(auth_code))
		pr('Authorizing, please be patient, it may take upto {} seconds...'.format(self._repr_timeout()))

		if self._use_server_auth:
			pars = {
				'code' : auth_code,
				'bypy_version' : const.__version__,
				'redirect_uri' : 'oob' }

			result = None
			# TODO: hacky
			global perr
			savedperr = perr
			if not self.debug:
				perr = nop

			self._update_auth_server_list()
			for auth in self.AuthServerList:
				(url, retry, msg) = auth
				pr(msg)
				result = self._get(url, pars, self._server_auth_act, retry = retry, addtoken = False)
				if result == const.ENoError:
					break
			if not self.debug:
				perr = savedperr

			if result == const.ENoError:
				pr("Successfully authorized")
			else:
				perr("Fatal: All server authorizations failed.")
				self._prompt_clean()
				sys.exit(result)
		else:
			pars = {
				'grant_type' : 'authorization_code',
				'code' : auth_code,
				'client_id' : self._apikey,
				'client_secret' : self._secretkey,
				'redirect_uri' : 'oob'
			}
			result = self._post(const.TokenUrl, pars, self._local_auth_act, addtoken = False)

		return result

	def _device_auth_act(self, r, args):
		dj = r.json()
		self.pd('device response: {}'.format(dj))
		return self._get_token(dj)

	def _device_auth(self):
		pars = {
			'client_id' : self._apikey,
			'response_type' : 'device_code',
			'scope' : 'basic netdisk'}
		return self._get(const.DeviceAuthUrl, pars, self._device_auth_act, addtoken = False)

	def _get_token_act(self, r, args):
		return self._store_json(r)

	def _get_token(self, deviceJson):
		# msg = "Please visit:{}\n" + deviceJson['verification_url'] + \
		# 	  "\nwithin " + str(deviceJson['expires_in']) + " seconds\n"
		# "Input the CODE: {}\n".format(deviceJson['user_code'])" + \
		# 	"and Authorize this little app.\n"
		msg = "Please visit:\n{}\nwithin {} seconds\n" \
			"Input the CODE: {}\n" \
			"and Authorize this little app.\n".format(
				deviceJson['verification_url'],
				str(deviceJson['expires_in']),
				deviceJson['user_code'])
		ask(msg)

		pars = {
			'grant_type' : 'device_token',
			'code' :  deviceJson['device_code'],
			'client_id' : self._apikey,
			'client_secret' : self._secretkey}

		return self._get(const.TokenUrl, pars, self._get_token_act, addtoken = False)

	def _refresh_token_act(self, r, args):
		return self._store_json(r)

	def _refresh_token(self):
		if self._use_server_auth:
			pr('Refreshing, please be patient, it may take upto {} seconds...'.format(self._repr_timeout()))

			pars = {
				'bypy_version' : const.__version__,
				'refresh_token' : self._json['refresh_token'] }

			result = None
			# TODO: hacky
			global perr
			savedperr = perr
			if not self.debug:
				perr = nop
			self._update_auth_server_list()
			for refresh in self.RefreshServerList:
				(url, retry, msg) = refresh
				pr(msg)
				result = self._get(url, pars, self._refresh_token_act, retry = retry, addtoken = False)
				if result == const.ENoError:
					break
			if not self.debug:
				perr = savedperr

			if result == const.ENoError:
				pr("Token successfully refreshed")
			else:
				perr("Token-refreshing on all the servers failed")
				self._prompt_clean()
				sys.exit(result)

			return result
		else:
			pars = {
				'grant_type' : 'refresh_token',
				'refresh_token' : self._json['refresh_token'],
				'client_secret' : self._secretkey,
				'client_id' : self._apikey}
			return self._post(const.TokenUrl, pars, self._refresh_token_act)

	def _walk_normal_file(self, dir):
		#dirb = dir.encode(FileSystemEncoding)
		for walk in os.walk(dir, followlinks=self._followlink):
			normalfiles = [t for t in walk[-1]
								if os.path.isfile(os.path.join(walk[0], t))]
			normalfiles.sort()
			normalwalk = walk[:-1] + (normalfiles,)
			yield normalwalk

	def _quota_act(self, r, args):
		j = r.json()
		self.jsonq.append(j)
		pr('Quota: ' + human_size(j['quota']))
		pr('Used: ' + human_size(j['used']))
		return const.ENoError

	@staticmethod
	def help(command): # this comes first to make it easy to spot
		''' Usage: help <command> - provide some information for the command '''
		for i, v in ByPy.__dict__.items():
			if callable(v) and v.__doc__ and v.__name__ == command :
				help = v.__doc__.strip()
				pos = help.find(const.HelpMarker)
				if pos != -1:
					pr("Usage: " + help[pos + len(const.HelpMarker):].strip())

	def refreshtoken(self):
		''' Usage: refreshtoken - refresh the access token '''
		return self._refresh_token()

	def _remove_remote_on_success(self, remotepath):
		if self._deletesource:
			self.pd("Removing remote path '{}' after successful download.".format(remotepath))
			result = self._delete(remotepath)
			if result == const.ENoError:
				self.pd("Remote path '{}' removed.".format(remotepath))
			else:
				perr("Failed to remove remote path '{}'.".format(remotepath))
			return result

	def _remove_local_on_success(self, localpath):
		if self._deletesource:
			self.pd("Removing local path '{}' after successful upload.".format(localpath))
			result = cached.remove_path_and_cache(localpath)
			if result == const.ENoError:
				self.pd("Local path '{}' removed.".format(localpath))
			else:
				perr("Failed to remove local path '{}'.".format(localpath))
			return result

	def info(self):
		return self.quota()

	def quota(self):
		''' Usage: quota/info - display the quota information '''
		pars = {
			'method' : 'info' }
		return self._get(pcsurl + 'quota', pars, self._quota_act)

	def _whoami_act(self, r, args):
		j = r.json()
		self.jsonq.append(j)
		self.pd("Response: {}".format(j))
		pr('User id: ' + j['userid'])
		pr('Username: ' + j['username'])
		return const.ENoError

	def whoami(self):
		''' Usage: whoami - display the user information '''
		pars = {}
		return self._get(const.OpenApiUrl + '/rest/2.0/passport/users/getInfo', pars, self._whoami_act)

	# return:
	#   0: local and remote files are of same size
	#   1: local file is larger
	#   2: remote file is larger
	#  -1: inconclusive (probably invalid remote json)
	def _compare_size(self, lsize, rjson):
		if 'size' in rjson:
			rsize = rjson['size']
			if lsize == rsize:
				return 0
			elif lsize > rsize:
				return 1
			else:
				return 2
		else:
			return -1

	def _verify_current_file(self, j, gotlmd5):
		# if we really don't want to verify
		if self._current_file == '/dev/null' and not self._verify:
			return const.ENoError

		rsize = 0
		rmd5 = 0

		# always perform size check even _verify is False
		if 'size' in j:
			rsize = j['size']
		else:
			perr("Unable to verify JSON: '{}', as no 'size' entry found".format(j))
			return const.EHashMismatch

		if 'md5' in j:
			rmd5 = j['md5']
		#elif 'block_list' in j and len(j['block_list']) > 0:
		#	rmd5 = j['block_list'][0]
		#else:
		#	# quick hack for meta's 'block_list' field
		#	pwarn("No 'md5' nor 'block_list' found in json:\n{}".format(j))
		#	pwarn("Assuming MD5s match, checking size ONLY.")
		#	rmd5 = self._current_file_md5
		else:
			perr("Unable to verify JSON: '{}', as no 'md5' entry found".format(j))
			return const.EHashMismatch

		self.pd("Comparing local file '{}' and remote file '{}'".format(
			self._current_file, j['path']))
		self.pd("Local file size : {}".format(self._current_file_size))
		self.pd("Remote file size: {}".format(rsize))

		if self._current_file_size == rsize:
			self.pd("Local and remote file size matches")
			if self._verify:
				if not gotlmd5:
					self._current_file_md5 = md5(self._current_file)
				self.pd("Local file MD5 : {}".format(self._current_file_md5))
				self.pd("Remote file MD5: {}".format(rmd5))

				if self._current_file_md5 == rmd5:
					self.pd("Local and remote file hash matches")
					return const.ENoError
				else:
					pinfo("Local and remote file hash DOESN'T match")
					return const.EHashMismatch
			else:
				return const.ENoError
		else:
			pinfo("Local and remote file size DOESN'T match")
			return const.EHashMismatch

	def _get_file_info_act(self, r, args):
		try:
			remotefile = args[0]
			j = r.json()
			self.jsonq.append(j)
			self.pd("List json: {}".format(j))
			if not 'list' in j:
				return const.EFileNotFound
			args[1] = len(j['list'])
			for f in j['list']:
				if f['path'] == remotefile: # case-sensitive
					self._remote_json = f
					self.pd("File info json: {}".format(self._remote_json))
					return const.ENoError

			return const.EFileNotFound
		except KeyError as ex:
			perr(formatex(ex))
			return const.ERequestFailed

	# the 'meta' command sucks, since it doesn't supply MD5 ...
	# now the JSON is written to self._remote_json, due to Python call-by-reference chaos
	# https://stackoverflow.com/questions/986006/python-how-do-i-pass-a-variable-by-reference
	# as if not enough confusion in Python call-by-reference
	def _get_file_info(self, remotefile, **kwargs):
		if remotefile == const.AppPcsPath: # root path
			# fake it
			rj = {}
			rj['isdir'] = 1
			rj['ctime'] = 0
			rj['fs_id'] = 0
			rj['mtime'] = 0
			rj['path'] = const.AppPcsPath
			rj['md5'] = ''
			rj['size'] = 0
			self._remote_json = rj
			self.pd("File info json: {}".format(self._remote_json))
			return const.ENoError

		rdir, rfile = posixpath.split(remotefile)
		self.pd("__get_file_info(): rdir : {} | rfile: {}".format(rdir, rfile))
        # while refactoring to use _walk_proceed_remote_dir is more dry, it might be less performant -
		# we want to stop once we found the file we need
		if rdir and rfile:
			listStart = 0
			while True:
				pars = {
					'method' : 'list',
					'path' : rdir,
					'by' : 'name', # sort in case we can use binary-search, etc in the futrue.
					'order' : 'asc',
					'limit': '{}-{}'.format(listStart, listStart + const.MaxListEntries)}
				reqargs = [remotefile, 0]
				result = self._get(pcsurl + 'file', pars, self._get_file_info_act, reqargs, **kwargs)
				if result == const.ENoError or reqargs[1] < const.MaxListEntries:
					break
				listStart += const.MaxListEntries
			return result
		else:
			perr("Invalid remotefile '{}' specified.".format(remotefile))
			return const.EArgument

	def get_file_info(self, remotefile = '/'):
		rpath = get_pcs_path(remotefile)
		return self._get_file_info(rpath)

	def _list_act(self, r, args):
		(remotedir, fmt) = args
		j = r.json()
		self.pd("Response: {}".format(j))
		self.jsonq.append(j)
		pr("{} ({}):".format(remotedir, fmt))
		for f in j['list']:
			pr(self._replace_list_format(fmt, f))

		return const.ENoError

	def _proceed_list(self, walkresult, remotepath, dirjs, filejs, args):
		result = walkresult
		fmt = args
		# cutfront = const.AppPcsPathLen
		cutfront = len(remotepath) + 1
		pathlol = [deepcopy(dirjs), deepcopy(filejs)]
		self.pd("Files listl: {}".format(pathlol))
		for path in pathlol:
			for p in path:
				if 'path' in p:
					p['path'] = p['path'][cutfront:]
					pr(self._replace_list_format(fmt, p))
		return result

	def ls(self, remotepath = '',
		fmt = '$t $f $s $m $d',
		sort = 'name', order = 'asc'):
		return self.list(remotepath, fmt, sort, order)

	def list(self, remotepath = '',
		fmt = '$t $f $s $m $d',
		sort = 'name', order = 'asc'):
		''' Usage: list/ls [remotepath] [format] [sort] [order] - list the 'remotepath' directory at Baidu PCS
	remotepath - the remote path at Baidu PCS. default: root directory '/'
	format - specifies how the list are displayed
	  $t - Type: Directory ('D') or File ('F')
	  $f - File name
	  $c - Creation time
	  $m - Modification time
	  $d - MD5 hash
	  $s - Size
	  Because of the interpolation, you need to escape the $ sign,
	  so '\$t - \$f - \$s - \$' will display "Type - File - Size - $'
	  Default format: '$t $f $s $m $d'
		'''
    # sort - sorting by [name, time, size]. default: 'name'
    # order - sorting order [asc, desc]. default: 'asc'
		rpath = get_pcs_path(remotepath)
		pr("{} ({}):".format(rpath, fmt))
		self.file_list = [] # Hacky, but too lazy to make it proper
		return self._walk_proceed_remote_dir(rpath, self._proceed_list, args = fmt, recursive = False)

		# pars = {
		# 	'method' : 'list',
		# 	'path' : rpath,
		# 	'by' : sort,
		# 	'order' : order }

		# return self._get(pcsurl + 'file', pars, self._list_act, (rpath, fmt))

	def _meta_act(self, r, args):
		return self._list_act(r, args)

	def _meta(self, rpath, fmt):
		pars = {
			'method' : 'meta',
			'path' : rpath }
		return self._get(pcsurl + 'file', pars,
			self._meta_act, (rpath, fmt))

	# multi-file meta is not implemented for its low usage
	def meta(self, remotepath = '/', fmt = '$t $u $f $s $c $m $i $b'):
		''' Usage: meta <remotepath> [format] - \
get information of the given path (dir / file) at Baidu Yun.
  remotepath - the remote path
  format - specifies how the list are displayed
    it supports all the format variables in the 'list' command, and additionally the followings:
	$i - fs_id
	$b - MD5 block_list
	$u - Has sub directory or not
'''
		rpath = get_pcs_path(remotepath)
		self.file_list = [] # Hacky, but too lazy to make it proper
		return self._meta(rpath, fmt)

	# this 'is_revision' parameter sometimes gives the following error (e.g. for rapidupload):
	# {u'error_code': 31066, u'error_msg': u'file does not exist'}
	# and maintain it is also an extra burden, so it's disabled for now
	def _add_isrev_param(self, ondup, pars):
		pass
		#if self._isrev and ondup != 'newcopy':
		#	pars['is_revision'] = 1

	def _combine_file_act(self, r, args):
		j = r.json()
		self.jsonq.append(j)
		result = self._verify_current_file(j, False)
		if result == const.ENoError:
			self.pv("'{}' =C=> '{}' OK.".format(self._current_file, args))
		else:
			perr("'{}' =C=> '{}' FAILED.".format(self._current_file, args))
		# save the md5 list, in case we add in resume function later to this program
		self._last_slice_md5s = self._slice_md5s
		self._slice_md5s = []

		return result

	def _combine_file(self, remotepath, ondup = 'overwrite'):
		pars = {
			'method' : 'createsuperfile',
			'path' : remotepath,
			'ondup' : ondup }
		self._add_isrev_param(ondup, pars)

		# always print this, so that we can use these data to combine file later
		pr("Combining the following MD5 slices:")
		for m in self._slice_md5s:
			pr(m)

		param = { 'block_list' : self._slice_md5s }
		return self._post(pcsurl + 'file',
				pars, self._combine_file_act,
				remotepath,
				data = { 'param' : json.dumps(param) } )

	def _upload_slice_act(self, r, args):
		j = r.json()
		# slices must be verified and re-upload if MD5s don't match,
		# otherwise, it makes the uploading slower at the end
		rsmd5 = j['md5']
		self.pd("Uploaded MD5 slice: " + rsmd5)
		if self._current_slice_md5 == rsmd5:
			self._slice_md5s.append(rsmd5)
			self.pv("'{}' >>==> '{}' OK.".format(self._current_file, args))
			return const.ENoError
		else:
			perr("'{}' >>==> '{}' FAILED.".format(self._current_file, args))
			return const.EHashMismatch

	# want to be proper? properness doesn't work
	# there seems to be a bug at Baidu's handling of http text:
	# Content-Disposition: ...  filename=utf-8''yourfile.ext
	# (pass '-ddd' to this program to verify this)
	# when you specify a unicode file name, which will be encoded
	# using the utf-8'' syntax
	# so, we put a work-around here: we always call our file 'file'
	# NOTE: an empty file name '' doesn't seem to work, so we
	# need to give it a name at will, but empty one.
	# apperantly, Baidu PCS doesn't use this file name for
	# checking / verification, so we are probably safe here.
	#files = { 'file' : (os.path.basename(self._current_file), self._current_slice) } )
	#files = { 'file' : (os.path.basename(localpath), f) })
	#files = { 'file' : ('file', f) })
	def _stream_upload(self, fo, pars, act, remotepath,
			url = cpcsurl + 'file',
			uploadfilename = 'file'):
		# https://stackoverflow.com/a/35784072/404271
		form = multipart.encoder.MultipartEncoder({
			'file': (uploadfilename, fo, 'application/octet-stream')})
		headers = {"Content-Type": form.content_type}
		return self._post(url,
			pars, act, remotepath,
			headers=headers, data=form)

	def _upload_slice(self, remotepath):
		pars = {
			'method' : 'upload',
			'type' : 'tmpfile'}

		return self._stream_upload(self._current_slice,
				pars, self._upload_slice_act, remotepath)

	def _update_progress_entry(self, fullpath):
		progress = {}

		try:
			progress = jsonload(self._progresspath)
		except Exception as ex:
			perr("Error loading the progress for: '{}'.\n{}.".format(fullpath, formatex(ex)))

		self.pd("Updating slice upload progress for {}".format(fullpath))
		progress[fullpath] = (self._slice_size, self._slice_md5s)

		try:
			jsondump(progress, self._progresspath, mpsemaphore)
		except Exception as ex:
			perr("Error updating the progress for: '{}'.\n{}.".format(fullpath, formatex(ex)))

	def _delete_progress_entry(self, fullpath):
		try:
			progress = jsonload(self._progresspath)
			# http://stackoverflow.com/questions/11277432/how-to-remove-a-key-from-a-python-dictionary
			#del progress[fullpath]
			self.pd("Removing slice upload progress for {}".format(fullpath))
			progress.pop(fullpath, None)
			jsondump(progress, self._progresspath, mpsemaphore)
		except Exception as ex:
			perr("Error deleting the progress for: '{}'.\n{}.".format(fullpath, formatex(ex)))

	def _upload_file_slices(self, localpath, remotepath, ondup = 'overwrite'):
		pieces = const.MaxSlicePieces
		slice = self._slice_size
		if self._current_file_size <= self._slice_size * const.MaxSlicePieces:
			# slice them using slice size
			pieces = (self._current_file_size + self._slice_size - 1 ) // self._slice_size
		else:
			# the following comparision is done in the caller:
			# elif self._current_file_size <= MaxSliceSize * MaxSlicePieces:

			# no choice, but need to slice them to 'MaxSlicePieces' pieces
			slice = (self._current_file_size + const.MaxSlicePieces - 1) // const.MaxSlicePieces

		self.pd("Slice size: {}, Pieces: {}".format(slice, pieces))

		i = 0
		ec = const.ENoError

		fullpath = os.path.abspath(self._current_file)
		progress = {}
		initial_offset = 0
		# create an empty progress file first
		if not os.path.exists(self._progresspath):
			try:
				jsondump(progress, self._progresspath, mpsemaphore)
			except Exception as ex:
				perr("Error savingprogress, no resumption.\n{}".format(formatex(ex)))

		try:
			progress = jsonload(self._progresspath)
		except Exception as ex:
			perr("Error loading progress, no resumption.\n{}".format(formatex(ex)))

		if fullpath in progress:
			self.pd("Find the progress entry resume uploading")
			(slice, md5s) = progress[fullpath]
			self._slice_md5s = []
			with io.open(self._current_file, 'rb') as f:
				self.pd("Verifying the md5s. Total count = {}".format(len(md5s)))
				for md in md5s:
					cslice = f.read(slice)
					cm = hashlib.md5(cslice)
					if (cm.hexdigest() == md):
						self.pd("{} verified".format(md))
						# TODO: a more rigorous check would be also verifying
						# slices exist at Baidu Yun as well (rapidupload test?)
						# but that's a bit complex. for now, we don't check
						# this but simply delete the progress entry if later
						# we got error combining the slices.
						self._slice_md5s.append(md)
					else:
						break
				self.pd("verified md5 count = {}".format(len(self._slice_md5s)))
			i = len(self._slice_md5s)
			initial_offset = i * slice
			self.pd("Start from offset {}".format(initial_offset))

		with io.open(self._current_file, 'rb') as f:
			start_time = time.time()
			f.seek(initial_offset, os.SEEK_SET)
			while i < pieces:
				self._current_slice = f.read(slice)
				m = hashlib.md5()
				m.update(self._current_slice)
				self._current_slice_md5 = m.hexdigest()
				self.pd("Uploading MD5 slice: {}, #{} / {}".format(
					self._current_slice_md5,
					i + 1, pieces))
				j = 0
				while True:
					ec = self._upload_slice(remotepath)
					if ec == const.ENoError:
						self.pd("Slice MD5 match, continuing next slice")
						pprgr(f.tell(), self._current_file_size, start_time, initial_offset)
						self._update_progress_entry(fullpath)
						break
					elif j < self._retry:
						j += 1
						# TODO: Improve or make it DRY with the _request retry logic
						perr("Slice MD5 mismatch, waiting {} seconds before retrying...".format(const.RetryDelayInSec))
						time.sleep(const.RetryDelayInSec)
						perr("Retrying #{} / {}".format(j + 1, self._retry))
					else:
						self._slice_md5s = []
						break
				if ec != const.ENoError:
					break
				i += 1

		if ec != const.ENoError:
			return ec
		else:
			#self.pd("Sleep 2 seconds before combining, just to be safer.")
			#time.sleep(2)
			ec = self._combine_file(remotepath, ondup = 'overwrite')
			if ec == const.ENoError \
				or ec == const.IESuperfileCreationFailed \
				or ec == const.IEBlockMissInSuperFile2 \
				or ec == const.EMaxRetry: # to handle undocumented error codes if any
				# we delete on success or failure caused by
				# the slices uploaded expired / became invalid
				# (needed for a fresh re-upload later)
				self._delete_progress_entry(fullpath)
			return ec

	def _rapidupload_file_act(self, r, args):
		if self._verify:
			self.pd("Not strong-consistent, sleep 1 second before verification")
			time.sleep(1)
			return self._verify_current_file(r.json(), True)
		else:
			return const.ENoError

	def _rapidupload_file_post(self, rpath, size, md5str, slicemd5str, crcstr, ondup = 'overwrite'):
		pars = {
			'method' : 'rapidupload',
			'path' : rpath,
			'content-length' : size,
			'content-md5' : md5str,
			'slice-md5' : slicemd5str,
			'content-crc32' : crcstr,
			'ondup' : ondup }
		self._add_isrev_param(ondup, pars)

		self.pd("RapidUploading Length: {} MD5: {}, Slice-MD5: {}, CRC: {}".format(
			size, md5str, slicemd5str, crcstr))
		return self._post(pcsurl + 'file', pars, self._rapidupload_file_act)

	def _get_hashes_for_rapidupload(self, lpath, setlocalfile = False):
		if setlocalfile:
			self._current_file = lpath
			self._current_file_size = getfilesize(lpath)

		self._current_file_md5 = md5(self._current_file)
		self._current_file_slice_md5 = slice_md5(self._current_file)
		self._current_file_crc32 = crc32(self._current_file)

	def _rapidupload_file(self, lpath, rpath, ondup = 'overwrite', setlocalfile = False):
		self._get_hashes_for_rapidupload(lpath, setlocalfile)

		md5str = self._current_file_md5
		slicemd5str =  self._current_file_slice_md5
		crcstr = hex(self._current_file_crc32)
		return self._rapidupload_file_post(rpath, self._current_file_size, md5str, slicemd5str, crcstr, ondup)

	def _upload_one_file_act(self, r, args):
		result = self._verify_current_file(r.json(), False)
		if result == const.ENoError:
			self.pv("'{}' ==> '{}' OK.".format(self._current_file, args))
		else:
			perr("'{}' ==> '{}' FAILED.".format(self._current_file, args))

		return result

	def _upload_one_file(self, localpath, remotepath, ondup = 'overwrite'):
		pars = {
			'method' : 'upload',
			'path' : remotepath,
			'ondup' : ondup }
		self._add_isrev_param(ondup, pars)

		with io.open(localpath, 'rb') as f:
			return self._stream_upload(f, pars,
					self._upload_one_file_act, remotepath)

	#TODO: upload empty directories as well?
	def _walk_upload(self, localpath, remotepath, ondup, walk):
		(dirpath, dirnames, filenames) = walk

		rdir = os.path.relpath(dirpath, localpath)
		if rdir == '.':
			rdir = ''
		else:
			rdir = rdir.replace('\\', '/')

		rdir = (remotepath + '/' + rdir).rstrip('/') # '/' bites

		result = const.ENoError
		for name in filenames:
			#lfile = os.path.join(dirpath, name)
			lfile = joinpath(dirpath, name)
			self._current_file = lfile
			self._current_file_size = getfilesize(lfile)
			rfile = rdir + '/' + name.replace('\\', '/')
			# if the corresponding file matches at Baidu Yun, then don't upload
			upload = True
			self._isrev = False
			self._remote_json = {}
			subresult = self._get_file_info(rfile, dumpex = False)
			if subresult == const.ENoError: # same-name remote file exists
				self._isrev = True
				if const.ENoError == self._verify_current_file(self._remote_json, False):
					# the two files are the same
					upload = False
					self.pv("Remote file '{}' already exists, skip uploading".format(rfile))
				else: # the two files are different
					if not self.shalloverwrite("Remote file '{}' exists but is different, "
							"do you want to overwrite it? [y/N]".format(rfile)):
						upload = False
				self._isrev = False

			if upload:
				fileresult = self._upload_file(lfile, rfile, ondup)
				if fileresult != const.ENoError:
					result = fileresult # we still continue
			else:
				pinfo("Remote file '{}' exists and is the same, skip uploading".format(rfile))
				# next / continue

		return result

	def _upload_dir_single(self, localpath, remotepath, ondup = 'overwrite'):
		result = const.ENoError
		# it's so minor that we don't care about the return value
		#self._mkdir(remotepath, retry = False, dumpex = False)
		#for walk in os.walk(localpath, followlinks=self._followlink):
		for walk in self._walk_normal_file(localpath):
			thisresult = self._walk_upload(localpath, remotepath, ondup, walk)
			# we continue even if some upload failed, but keep the last error code
			if thisresult != const.ENoError:
				result = thisresult

		return result

	def _upload_dir_multi(self, localpath, remotepath, ondup = 'overwrite'):
		worker = partial(self._walk_upload, localpath, remotepath, ondup)
		return self._multi_process(worker, self._walk_normal_file(localpath),
			"Direcory Upload")

	def _upload_dir(self, localpath, remotepath, ondup = 'overwrite'):
		self.pd("Uploading directory '{}' to '{}'".format(localpath, remotepath))
		result = const.ENoError
		if Pool and self.processes > 1:
			result = self._upload_dir_multi(localpath, remotepath, ondup)
		else:
			result =  self._upload_dir_single(localpath, remotepath, ondup)
		if result == const.ENoError:
			self._remove_local_on_success(localpath)
		return result

	def _upload_file(self, localpath, remotepath, ondup = 'overwrite'):
		# TODO: this is a quick patch
		if not self._shallinclude(localpath, remotepath, True):
			# since we are not going to upload it, there is no error
			#return const.ENoError
			return const.ESkipped

		self._current_file = localpath
		self._current_file_size = getfilesize(localpath)

		result = const.ENoError
		if self._current_file_size > const.MinRapidUploadFileSize:
			self.pd("'{}' is being RapidUploaded.".format(self._current_file))
			result = self._rapidupload_file(localpath, remotepath, ondup)
			if result == const.ENoError:
				self.pv("RapidUpload: '{}' =R=> '{}' OK.".format(localpath, remotepath))
				self._rapiduploaded = True
			else:
				self._rapiduploaded = False
				if not self._rapiduploadonly:
					self.pd("'{}' can't be RapidUploaded, now trying normal uploading.".format(
						self._current_file))
					# rapid upload failed, we have to upload manually
					if self._current_file_size <= self._slice_size:
						self.pd("'{}' is being non-slicing uploaded.".format(self._current_file))
						# no-slicing upload
						result = self._upload_one_file(localpath, remotepath, ondup)
					elif self._current_file_size <= const.MaxSliceSize * const.MaxSlicePieces:
						# slice them using slice size
						self.pd("'{}' is being slicing uploaded.".format(self._current_file))
						result = self._upload_file_slices(localpath, remotepath, ondup)
					else:
						result = const.EFileTooBig
						perr("Error: size of file '{}' - {} is too big".format(
							self._current_file,
							self._current_file_size))
				else:
					self.pv("'{}' can't be rapidly uploaded, so it's skipped since we are in the rapid-upload-only mode.".format(localpath))
					result = const.ESkipped
		elif self._rapiduploadonly:
			self.pv("'{}' is too small to be rapidly uploaded, so it's skipped since we are in the rapid-upload-only mode.".format(localpath))
			result = const.ESkipped
		else:
			# very small file, must be uploaded manually and no slicing is needed
			self.pd("'{}' is small and being non-slicing uploaded.".format(self._current_file))
			result = self._upload_one_file(localpath, remotepath, ondup)

		if result == const.ENoError:
			self._remove_local_on_success(localpath)
		return result

	def upload(self, localpath = '', remotepath = '', ondup = "overwrite"):
		''' Usage: upload [localpath] [remotepath] [ondup] - \
upload a file or directory (recursively)
    localpath - local path, is the current directory '.' if not specified
    remotepath - remote path at Baidu Yun (after app root directory at Baidu Yun)
    ondup - what to do upon duplication ('overwrite' or 'newcopy'), default: 'overwrite'
		'''
		# copying since Python is call-by-reference by default,
		# so we shall not modify the passed-in parameters
		lpath = localpath.rstrip('\\/ ') # no trailing slashes
		lpathbase = os.path.basename(lpath)
		rpath = remotepath
		if not lpath:
			# so, if you don't specify the local path, it will always be the current directory
			# and thus isdir(localpath) is always true
			lpath = os.path.abspath(".")
			self.pd("localpath not set, set it to current directory '{}'".format(localpath))

		if os.path.isfile(lpath):
			self.pd("Uploading file '{}'".format(lpath))
			if not rpath or rpath == '/': # to root we go
				rpath = lpathbase
			if rpath[-1] == '/': # user intends to upload to this DIR
				rpath = get_pcs_path(rpath + lpathbase)
			else:
				rpath = get_pcs_path(rpath)
				# avoid uploading a file and destroy a directory by accident
				subresult = self._get_file_info(rpath)
				if subresult == const.ENoError: # remote path exists, check is dir or file
					if self._remote_json['isdir']: # do this only for dir
						rpath += '/' + lpathbase # rpath is guaranteed no '/' ended
			self.pd("remote path is '{}'".format(rpath))
			return self._upload_file(lpath, rpath, ondup)
		elif os.path.isdir(lpath):
			self.pd("Uploading directory '{}' recursively".format(lpath))
			rpath = get_pcs_path(rpath)
			return self._upload_dir(lpath, rpath, ondup)
		else:
			perr("Error: invalid local path '{}' for uploading specified.".format(localpath))
			return const.EParameter

	# The parameter 'localfile' is a bit kluge as it carries double meanings,
	# but this is to be command line friendly (one can not input an empty string '' from the command line),
	# so let's just leave it like this unless we can devise a cleverer / clearer weay
	def combine(self, remotefile, localfile = '*', *args):
		''' Usage: combine <remotefile> [localfile] [md5s] - \
try to create a file at PCS by combining slices, having MD5s specified
  remotefile - remote file at Baidu Yun (after app root directory at Baidu Yun)
  localfile - local file to verify against, passing in a star '*' or '/dev/null' means no verification
  md5s - MD5 digests of the slices, can be:
    - list of MD5 hex strings separated by spaces
    - a string in the form of 'l<path>' where <path> points to a text file containing MD5 hex strings separated by spaces or line-by-line
		'''
		self._slice_md5s = []
		if args:
			if args[0].upper() == 'L':
				try:
					with io.open(args[1:], 'r', encoding = 'utf-8') as f:
						contents = f.read()
						digests = filter(None, contents.split())
						for d in digests:
							self._slice_md5s.append(d)
				except IOError as ex:
					perr("Exception occured while reading file '{}'.\n{}".format(
						localfile, formatex(ex)))
			else:
				for arg in args:
					self._slice_md5s.append(arg)
		else:
			perr("You MUST provide the MD5s hex strings through arguments or a file.")
			return const.EArgument

		original_verify = self._verify
		if not localfile or localfile == '*' or localfile == '/dev/null':
			self._current_file = '/dev/null' # Force no verify
			self._verify = False
		else:
			self._current_file = localfile
			self._current_file_size = getfilesize(localfile)

		result = self._combine_file(get_pcs_path(remotefile))
		self._verify = original_verify
		return result

	# no longer used
	def _get_meta_act(self, r, args):
		parse_ok = False
		j = r.json()
		self.jsonq.append(j)
		if 'list' in j:
			lj = j['list']
			if len(lj) > 0:
				self._remote_json = lj[0] # TODO: ugly patch
				# patch for inconsistency between 'list' and 'meta' json
				#self._remote_json['md5'] = self._remote_json['block_list'].strip('[]"')
				self.pd("self._remote_json: {}".format(self._remote_json))
				parse_ok = True
				return const.ENoError

		if not parse_ok:
			self._remote_json = {}
			perr("Invalid JSON:\n{}".format(j))
			return const.EInvalidJson

	# no longer used
	def _get_meta(self, remotefile):
		pars = {
			'method' : 'meta',
			'path' : remotefile }
		return self._get(
			pcsurl + 'file', pars,
			self._get_meta_act)

	# NO LONGER IN USE
	def _downfile_act(self, r, args):
		rfile, offset = args
		with io.open(self._current_file, 'r+b' if offset > 0 else 'wb') as f:
			if offset > 0:
				f.seek(offset)

			rsize = self._remote_json['size']
			start_time = time.time()
			for chunk in r.iter_content(chunk_size = self._dl_chunk_size):
				if chunk: # filter out keep-alive new chunks
					f.write(chunk)
					f.flush()
					pprgr(f.tell(), rsize, start_time)
					# https://stackoverflow.com/questions/7127075/what-exactly-the-pythons-file-flush-is-doing
					#os.fsync(f.fileno())

		# No exception above, then everything goes fine
		result = const.ENoError
		if self._verify:
			self._current_file_size = getfilesize(self._current_file)
			result = self._verify_current_file(self._remote_json, False)

		if result == const.ENoError:
			self.pv("'{}' <== '{}' OK".format(self._current_file, rfile))
		else:
			perr("'{}' <== '{}' FAILED".format(self._current_file, rfile))

		return result

	def _downchunks_act(self, r, args):
		rfile, offset, rsize, start_time = args

		expectedBytes = self._dl_chunk_size
		if rsize - offset < self._dl_chunk_size:
			expectedBytes = rsize - offset

		if len(r.content) != expectedBytes:
			return const.ERequestFailed
		else:
			with io.open(self._current_file, 'r+b' if offset > 0 else 'wb') as f:
				if offset > 0:
					f.seek(offset)

				f.write(r.content)
				pos = f.tell()
				pprgr(pos, rsize, start_time, existing = self._existing_size)
				if pos - offset == expectedBytes:
					return const.ENoError
				else:
					return const.EFileWrite

	def _down_aria2c(self, remotefile, localfile):
		url = "{}{}".format(dpcsurl, "file")
		# i think encoding in UTF-8 before escaping is presumably the best practice
		# http://stackoverflow.com/a/913653/404271
		pars = {
			"method": "download",
			"path": remotefile.encode('utf-8'),
			"access_token": self._access_token,
			}
		full_url = "{}?{}".format(url, ulp.urlencode(pars))
		cmd = ['aria2c', '--user-agent="{}"'.format(const.UserAgent)] \
			+ shlex.split(self._downloader_args) \
			+ ['-o', localfile, full_url]

		tries = self._retry
		subret = 0
		i = 0
		while True:
			self.pd("call: {}".format(cmd))
			subret = subprocess.call(cmd)
			self.pd("aria2c exited with status: {}".format(subret))
			if subret == 0:
				return const.ENoError
			i += 1
			if i < tries:
				# algo changed: delay more after each failure
				delay = const.RetryDelayInSec * i
				perr("Waiting {} seconds before retrying...".format(delay))
				time.sleep(delay)
				perr("Request Try #{} / {}".format(i + 1, tries))
			else:
				perr("Maximum number ({}) of tries failed.".format(tries))
				if self._quit_when_fail:
					sys.exit(const.EMaxRetry)
				return const.EMaxRetry

	# requirement: self._remote_json is already gotten
	def _downchunks(self, rfile, start):
		rsize = self._remote_json['size']

		pars = {
			'method' : 'download',
			# Do they cause some side effects?
			#'app_id': 250528,
			#'check_blue' : '1',
			#'ec' : '1',
			'path' : rfile }

		offset = start
		self._existing_size = offset
		start_time = time.time()
		while True:
			nextoffset = offset + self._dl_chunk_size
			if nextoffset < rsize:
				headers = { "Range" : "bytes={}-{}".format(
					offset, nextoffset - 1) }
			elif offset > 0:
				headers = { "Range" : "bytes={}-".format(offset) }
			elif rsize >= 1: # offset == 0
				# Fix chunked + gzip response,
				# seems we need to specify the Range for the first chunk as well:
				# https://github.com/houtianze/bypy/pull/161
				#headers = { "Range" : "bytes=0-".format(rsize - 1) }
				headers = { "Range" : "bytes=0-{}".format(rsize - 1) }
			else:
				headers = {}

			# this _may_ solve #163: { "error_code":31326, "error_msg":"anti hotlinking"}
#			if 'Range' in headers:
#				rangemagic = base64.standard_b64encode(headers['Range'][6:].encode('utf-8'))
#				self.pd("headers['Range'][6:]: {} {}".format(headers['Range'][6:], rangemagic))
#				#pars['ru'] = rangemagic

			subresult = self._get(dpcsurl + 'file', pars,
				self._downchunks_act, (rfile, offset, rsize, start_time), headers = headers, cookies = self._cookies)
			if subresult != const.ENoError:
				return subresult

			if nextoffset < rsize:
				offset += self._dl_chunk_size
			else:
				break

		# No exception above, then everything goes fine
		result = const.ENoError
		if self._verify:
			self._current_file_size = getfilesize(self._current_file)
			result = self._verify_current_file(self._remote_json, False)

		if result == const.ENoError:
			self.pv("'{}' <== '{}' OK".format(self._current_file, rfile))
		else:
			perr("'{}' <== '{}' FAILED".format(self._current_file, rfile))

		return result

	def _downfile(self, remotefile, localfile):
		pinfo('{} <- {}'.format(localfile, remotefile))
		# TODO: this is a quick patch
		if not self._shallinclude(localfile, remotefile, False):
			# since we are not going to download it, there is no error
			#return const.ENoError
			return const.ESkipped

		result = const.ENoError
		rfile = remotefile

		self._remote_json = {}
		self.pd("Downloading '{}' as '{}'".format(rfile, localfile))
		self._current_file = localfile
		#if self._verify or self._resumedownload:
		self.pd("Getting info of remote file '{}' for later verification".format(rfile))
		result = self._get_file_info(rfile)
		if result != const.ENoError:
			return result

		offset = 0
		self.pd("Checking if we already have the copy locally")
		if os.path.isfile(localfile):
			self.pd("Same-name local file '{}' exists, checking if contents match".format(localfile))
			self._current_file_size = getfilesize(self._current_file)
			if const.ENoError == self._verify_current_file(self._remote_json, False) \
				and not (self._downloader[:5] == const.DownloaderAria2 and os.path.exists(localfile + '.aria2')):
				self.pd("Same local file '{}' already exists, skip downloading".format(localfile))
				self._remove_remote_on_success(remotefile)
				return const.ENoError
			else:
				if not self.shalloverwrite("Same-name locale file '{}' exists but is different, "
						"do you want to overwrite it? [y/N]".format(localfile)):
					pinfo("Same-name local file '{}' exists but is different, skip downloading".format(localfile))
					#return const.ENoError
					return const.ESkipped

			if self._resumedownload and \
				self._compare_size(self._current_file_size, self._remote_json) == 2:
				if self._resumedl_revertcount < 0:
					if self._current_file_size:
						offset = self._current_file_size
				else:
					# revert back at least self._resumedl_revertcount download chunk(s), default: one
					pieces = self._current_file_size // self._dl_chunk_size
					if pieces > self._resumedl_revertcount:
						offset = (pieces - self._resumedl_revertcount) * self._dl_chunk_size
		elif os.path.isdir(localfile):
			if not self.shalloverwrite("Same-name directory '{}' exists, "
				"do you want to remove it? [y/N]".format(localfile)):
				pinfo("Same-name directory '{}' exists, skip downloading".format(localfile))
				#return const.ENoError
				return const.ESkipped

			self.pv("Directory with the same name '{}' exists, removing ...".format(localfile))
			result = removedir(localfile, self.verbose)
			if result == const.ENoError:
				self.pv("Removed")
			else:
				perr("Error removing the directory '{}'".format(localfile))
				return result

		ldir, file = os.path.split(localfile)
		if ldir and not os.path.exists(ldir):
			result = makedir(ldir, verbose = self.verbose)
			if result != const.ENoError:
				perr("Fail to make directory '{}'".format(ldir))
				return result

		if self._downloader[:5] == const.DownloaderAria2:
			result = self._down_aria2c(rfile, localfile)
		else:
			result = self._downchunks(rfile, offset)

		if result == const.ENoError:
			self._remove_remote_on_success(remotefile)

		return result

	def downfile(self, remotefile, localpath = ''):
		''' Usage: downfile <remotefile> [localpath] - \
download a remote file.
  remotefile - remote file at Baidu Yun (after app root directory at Baidu Yun)
  localpath - local path.
    if it ends with '/' or '\\', it specifies the local directory
    if it specifies an existing directory, it is the local directory
    if not specified, the local directory is the current directory '.'
    otherwise, it specifies the local file name
To stream a file using downfile, you can use the 'mkfifo' trick with omxplayer etc.:
  mkfifo /tmp/omx
  bypy.py downfile <remotepath> /tmp/omx &
  omxplayer /tmp/omx
		'''
		localfile = localpath
		if not localpath:
			localfile = os.path.basename(remotefile)
		elif localpath[-1] == '\\' or \
			localpath[-1] == '/' or \
			os.path.isdir(localpath):
			#localfile = os.path.join(localpath, os.path.basename(remotefile))
			localfile = joinpath(localpath, os.path.basename(remotefile))
		else:
			localfile = localpath

		pcsrpath = get_pcs_path(remotefile)
		return self._downfile(pcsrpath, localfile)

	def _stream_act_actual(self, r, args):
		pipe, csize = args
		with io.open(pipe, 'wb') as f:
			for chunk in r.iter_content(chunk_size = csize):
				if chunk: # filter out keep-alive new chunks
					f.write(chunk)
					f.flush()
					# https://stackoverflow.com/questions/7127075/what-exactly-the-pythons-file-flush-is-doing
					#os.fsync(f.fileno())

	def _streaming_act(self, r, args):
		return self._stream_act_actual(r, args)

	# NOT WORKING YET
	def streaming(self, remotefile, localpipe, fmt = 'M3U8_480_360', chunk = 4 * const.OneM):
		''' Usage: stream <remotefile> <localpipe> [format] [chunk] - \
stream a video / audio file converted to M3U format at cloud side, to a pipe.
  remotefile - remote file at Baidu Yun (after app root directory at Baidu Yun)
  localpipe - the local pipe file to write to
  format - output video format (M3U8_320_240 | M3U8_480_224 | \
M3U8_480_360 | M3U8_640_480 | M3U8_854_480). Default: M3U8_480_360
  chunk - chunk (initial buffering) size for streaming (default: 4M)
To stream a file, you can use the 'mkfifo' trick with omxplayer etc.:
  mkfifo /tmp/omx
  bypy.py downfile <remotepath> /tmp/omx &
  omxplayer /tmp/omx
  *** NOT WORKING YET ****
		'''
		pars = {
			'method' : 'streaming',
			'path' : get_pcs_path(remotefile),
			'type' : fmt }

		return self._get(pcsurl + 'file', pars,
			self._streaming_act, (localpipe, chunk), stream = True)

	def _walk_proceed_remote_dir_act(self, r, args):
		dirjs, filejs = args
		j = r.json()
		if 'list' not in j:
			self.pd("Key 'list' not found in the response of directory listing request:\n{}".format(j))
			return const.ERequestFailed

		paths = j['list']
		for path in paths:
			if path['isdir']:
				dirjs.append(path)
			else:
				filejs.append(path)

		return const.ENoError

	def _walk_remote_dir(self, remotepath, remoterootpath, args = None, skip_remote_only_dirs = False, recursive = True):
		dirjs = []
		filejs = []
		listStart = 0
		# https://github.com/houtianze/bypy/issues/285
		while True:
			pars = {
				'method' : 'list',
				'path' : remotepath,
				'by' : 'name',
				'order' : 'asc',
				'limit' : '{}-{}'.format(listStart, listStart + const.MaxListEntries)}
			# Python parameters are by-reference and mutable, so they are 'out' by default
			result = self._get(pcsurl + 'file', pars, self._walk_proceed_remote_dir_act, (dirjs, filejs))
			if len(dirjs) + len(filejs) - listStart < const.MaxListEntries:
				break
			listStart += const.MaxListEntries
		yield [result, remotepath, dirjs, filejs, args]
		if result == const.ENoError:
			self.pd("Remote dirs: {}".format(dirjs))
			self.pd("Remote files: {}".format(filejs))
			if recursive:
				for dirj in dirjs:
					crpath = dirj['path'] # crpath - current remote path
					if skip_remote_only_dirs and remoterootpath != None and \
						self._local_dir_contents.get(posixpath.relpath(crpath, remoterootpath)) == None:
						self.pd("Skipping remote-only sub-directory '{}'.".format(crpath))
						continue

					# http://stackoverflow.com/a/8991864/404271
					# for Python 3.3+, "yield from" reads better
					for y in self._walk_remote_dir(crpath, remoterootpath, args, skip_remote_only_dirs):
						yield y

	def _walk_proceed_remote_dir(self, remotepath, proceed, args = None, skip_remote_only_dirs = False, recursive = True):
		result = const.ENoError
		for walk in self._walk_remote_dir(remotepath, remotepath, args, skip_remote_only_dirs, recursive):
			subresult = proceed(*walk)
			if subresult != const.ENoError:
				result = subresult
		return result

	def _prepare_local_dir(self, localdir):
		result = const.ENoError
		if os.path.isfile(localdir):
			result = removefile(localdir, self.verbose)

		if result == const.ENoError:
			if localdir and not os.path.exists(localdir):
				result = makedir(localdir, verbose = self.verbose)

		return result

	def _proceed_downdir(self, walkresult, remotepath, dirjs, filejs, args):
		result = walkresult
		rootrpath, localpath = args
		#rlen = len(remotepath) + 1
		rootlen = len(rootrpath) + 1 # '+ 1' for the trailing '/', it bites.

		subresult = self._prepare_local_dir(localpath)
		if subresult != const.ENoError:
			perr("Fail to prepare local directory '{}' for downloading, ABORT!".format(localpath))
			return subresult

		for dirj in dirjs:
			rdir = dirj['path']
			reldir = rdir[rootlen:]
			#ldir = os.path.join(localpath, reldir)
			ldir = joinpath(localpath, reldir)
			subresult = self._prepare_local_dir(ldir)
			if subresult != const.ENoError:
				perr("Fail to prepare local directory '{}' for downloading, ABORT!".format(ldir))
				return subresult

		for filej in filejs:
			rfile = filej['path']
			relfile = rfile[rootlen:]
			#lfile = os.path.join(localpath, relfile)
			lfile = joinpath(localpath, relfile)
			subresult = self._downfile(rfile, lfile)
			if subresult != const.ENoError:
				result = subresult

		return result

	def _downdir_single(self, rpath, lpath):
		return self._walk_proceed_remote_dir(rpath, self._proceed_downdir, (rpath, lpath))

	def _proceed_downdir_worker(self, arglist):
		return self._proceed_downdir(*arglist)

	def _downdir_multi(self, rpath, lpath):
		return self._multi_process(self._proceed_downdir_worker,
			self._walk_remote_dir(rpath, rpath, (rpath, lpath)),
			"Directory Download")

	def _downdir(self, rpath, lpath):
		result = const.ENoError
		if Pool and self.processes > 1:
			result = self._downdir_multi(rpath, lpath)
		else:
			result = self._downdir_single(rpath, lpath)
		if result == const.ENoError:
			self._remove_remote_on_success(rpath)
		return result

	def downdir(self, remotepath = None, localpath = None):
		''' Usage: downdir [remotedir] [localdir] - \
download a remote directory (recursively)
  remotedir - remote directory at Baidu Yun (after app root directory), if not specified, it is set to the root directory at Baidu Yun
  localdir - local directory. if not specified, it is set to the current directory
		'''
		rpath = get_pcs_path(remotepath)
		lpath = localpath
		if not lpath:
			lpath = '' # empty string does it, no need '.'
		lpath = lpath.rstrip('/\\ ')
		return self._downdir(rpath, lpath)

	def download(self, remotepath = '/', localpath = ''):
		''' Usage: download [remotepath] [localpath] - \
download a remote directory (recursively) / file
  remotepath - remote path at Baidu Yun (after app root directory), if not specified, it is set to the root directory at Baidu Yun
  localpath - local path. if not specified, it is set to the current directory
		'''
		subr = self.get_file_info(remotepath)
		if const.ENoError == subr:
			if 'isdir' in self._remote_json:
				if self._remote_json['isdir']:
					return self.downdir(remotepath, localpath)
				else:
					return self.downfile(remotepath, localpath)
			else:
				perr("Malformed path info JSON '{}' returned".format(self._remote_json))
				return const.EFatal
		elif const.EFileNotFound == subr:
			perr("Remote path '{}' does not exist".format(remotepath))
			return subr
		else:
			perr("Error {} while getting info for remote path '{}'".format(subr, remotepath))
			return subr

	def _mkdir_act(self, r, args):
		if self.verbose:
			j = r.json()
			self.jsonq.append(j)
			pr("path, ctime, mtime, fs_id")
			pr("{path}, {ctime}, {mtime}, {fs_id}".format(**j))

		return const.ENoError

	def _mkdir(self, rpath, **kwargs):
		# TODO: this is a quick patch
		# the code still works because Baidu Yun doesn't require
		# parent directory to exist remotely to upload / create a file
		if not self._shallinclude('.', rpath, True):
			#return const.ENoError
			return const.ESkipped

		self.pd("Making remote directory '{}'".format(rpath))

		pars = {
			'method' : 'mkdir',
			'path' : rpath }
		return self._post(pcsurl + 'file', pars, self._mkdir_act, **kwargs)

	def mkdir(self, remotepath):
		''' Usage: mkdir <remotedir> - \
create a directory at Baidu Yun
  remotedir - the remote directory
'''
		rpath = get_pcs_path(remotepath)
		return self._mkdir(rpath)

	def _move_act(self, r, args):
		j = r.json()
		self.jsonq.append(j)
		list = j['extra']['list']
		fromp = list[0]['from']
		to = list[0]['to']
		self.pd("Remote move: '{}' =mm-> '{}' OK".format(fromp, to))

	# aliases
	def mv(self, fromp, to):
		return self.move(fromp, to)

	def rename(self, fromp, to):
		return self.move(fromp, to)

	def ren(self, fromp, to):
		return self.move(fromp, to)

	def move(self, fromp, to):
		''' Usage: move/mv/rename/ren <from> <to> - \
move a file / dir remotely at Baidu Yun
  from - source path (file / dir)
  to - destination path (file / dir)
		'''
		frompp = get_pcs_path(fromp)
		top = get_pcs_path(to)
		pars = {
			'method' : 'move',
			'from' : frompp,
			'to' : top }

		self.pd("Remote moving: '{}' =mm=> '{}'".format(fromp, to))
		return self._post(pcsurl + 'file', pars, self._move_act)

	def _copy_act(self, r, args):
		j = r.json()
		self.jsonq.append(j)
		for list in j['extra']['list']:
			fromp = list['from']
			to = list['to']
			self.pd("Remote copy: '{}' =cc=> '{}' OK".format(fromp, to))

		return const.ENoError

	# alias
	def cp(self, fromp, to):
		return self.copy(fromp, to)

	def copy(self, fromp, to):
		''' Usage: copy/cp <from> <to> - \
copy a file / dir remotely at Baidu Yun
  from - source path (file / dir)
  to - destination path (file / dir)
		'''
		frompp = get_pcs_path(fromp)
		top = get_pcs_path(to)
		pars = {
			'method' : 'copy',
			'from' : frompp,
			'to' : top }

		self.pd("Remote copying '{}' =cc=> '{}'".format(frompp, top))
		return self._post(pcsurl + 'file', pars, self._copy_act)

	def _delete_act(self, r, args):
		j = r.json()
		self.jsonq.append(j)
		rid = j['request_id']
		if rid:
			pr("Deletion request '{}' OK".format(rid))
			pr("Use 'list' command to confirm")

			return const.ENoError
		else:
			perr("Deletion failed")
			return const.EFailToDeleteFile

	def _delete(self, rpath):
		pars = {
			'method' : 'delete',
			'path' : rpath }

		self.pd("Remote deleting: '{}'".format(rpath))
		return self._post(pcsurl + 'file', pars, self._delete_act)

	def _delete_children_act(self, r, args):
		result = const.ENoError
		j = r.json()
		for f in j['list']:
			# we continue even if some upload failed, but keep the last error code
			thisresult = self._delete(f['path'])
			if thisresult != const.ENoError:
				result = thisresult

		return result

	def _delete_children(self, rpath):
		pars = {
			'method' : 'list',
			'path' : rpath}

		return self._get(pcsurl + 'file', pars, self._delete_children_act, None)

	# aliases
	def remove(self, remotepath):
		return self.delete(remotepath)

	def rm(self, remotepath):
		return self.delete(remotepath)

	def delete(self, remotepath):
		''' Usage: delete/remove/rm <remotepath> - \
delete a file / dir remotely at Baidu Yun
  remotepath - destination path (file / dir)
		'''
		rpath = get_pcs_path(remotepath)
		#if is_pcs_root_path(rpath):
		#	return self._delete_children(rpath)
		#else:
		return self._delete(rpath)

	def _search_act(self, r, args):
		j = r.json()
		self.jsonq.append(j)
		print_pcs_list(j)
		return const.ENoError

	def search(self, keyword, remotepath = None, recursive = True):
		''' Usage: search <keyword> [remotepath] [recursive] - \
search for a file using keyword at Baidu Yun
  keyword - the keyword to search
  remotepath - remote path at Baidu Yun, if not specified, it's app's root directory
  resursive - search recursively or not. default is true
		'''
		rpath = get_pcs_path(remotepath)

		pars = {
			'method' : 'search',
			'path' : rpath,
			'wd' : keyword,
			're' : '1' if str2bool(recursive) else '0'}

		self.pd("Searching: '{}'".format(rpath))
		return self._get(pcsurl + 'file', pars, self._search_act)

	def _listrecycle_act(self, r, args):
		print_pcs_list(r.json())
		return const.ENoError

	def listrecycle(self, start = 0, limit = 1000):
		''' Usage: listrecycle [start] [limit] - \
list the recycle contents
  start - starting point, default: 0
  limit - maximum number of items to display. default: 1000
		'''
		pars = {
			'method' : 'listrecycle',
			'start' : str2int(start),
			'limit' : str2int(limit) }

		self.pd("Listing recycle '{}'")
		return self._get(pcsurl + 'file', pars, self._listrecycle_act)

	def _restore_act(self, r, args):
		path = args
		pr("'{}' found and restored".format(path))
		return const.ENoError

	def _restore_search_act(self, r, args):
		path = args
		flist = r.json()['list']
		fsid = None
		for f in flist:
			if os.path.normpath(f['path'].lower()) == os.path.normpath(path.lower()):
				fsid = f['fs_id']
				self.pd("fs_id for restoring '{}' found".format(fsid))
				break
		if fsid:
			pars = {
				'method' : 'restore',
				'fs_id' : fsid }
			return self._post(pcsurl + 'file', pars, self._restore_act, path)
		else:
			perr("'{}' not found in the recycle bin".format(path))

	def restore(self, remotepath):
		''' Usage: restore <remotepath> - \
restore a file from the recycle bin
  remotepath - the remote path to restore
		'''
		rpath = get_pcs_path(remotepath)
		# by default, only 1000 items, more than that sounds a bit crazy
		pars = {
			'method' : 'listrecycle' }

		self.pd("Searching for fs_id to restore")
		return self._get(pcsurl + 'file', pars, self._restore_search_act, rpath)

	def _proceed_local_gather(self, dirlen, walk):
		#names.sort()
		(dirpath, dirnames, filenames) = walk

		files = []
		for name in filenames:
			#fullname = os.path.join(dirpath, name)
			fullname = joinpath(dirpath, name)
			# ignore broken symbolic links
			if not os.path.exists(fullname):
				self.pd("Local path '{}' does not exist (broken symbolic link?)".format(fullname))
				continue
			files.append((name, getfilesize(fullname), md5(fullname)))

		reldir = dirpath[dirlen:].replace('\\', '/')
		place = self._local_dir_contents.get(reldir)
		for dir in dirnames:
			place.add(dir, PathDictTree('D'))
		for file in files:
			place.add(file[0], PathDictTree('F', size = file[1], md5 = file[2]))

		return const.ENoError

	def _gather_local_dir(self, dir):
		self._local_dir_contents = PathDictTree()
		#for walk in os.walk(dir, followlinks=self._followlink):
		for walk in self._walk_normal_file(dir):
			self._proceed_local_gather(len(dir), walk)
		self.pd(self._local_dir_contents)

	def _proceed_remote_gather(self, walkresult, remotepath, dirjs, filejs, args = None):
		# NOTE: the '+ 1' is due to the trailing slash '/'
		# be careful about the trailing '/', it bit me once, bitterly
		rootrdir = args
		rootlen = len(rootrdir)
		dlen = len(remotepath) + 1
		for d in dirjs:
			self._remote_dir_contents.get(remotepath[rootlen:]).add(
				d['path'][dlen:], PathDictTree('D', size = d['size'], md5 = d['md5'] if 'md5' in d else ''))

		# Baidu made another fuck up here:
		# f['md5'] doesn't have the correct MD5 value, but f['block_list'][0] has
		# This makes no sense, and I'm not going to change the correct code to adapt its wrong behaviors
		# --- Code below for reference ---
		# fmd5 = f['md5']
		# bl = 'block_list'
		# if bl in f and f[bl]:
		# 	fmd5 = f[bl][0]
		# f['path'][dlen:], PathDictTree('F', size = f['size'], md5 = fmd5))
		for f in filejs:
			self._remote_dir_contents.get(remotepath[rootlen:]).add(
				f['path'][dlen:], PathDictTree('F', size = f['size'], md5 = f['md5']))

		return walkresult

	def _gather_remote_dir(self, rdir, skip_remote_only_dirs = False):
		self._remote_dir_contents = PathDictTree()
		self._walk_proceed_remote_dir(rdir, self._proceed_remote_gather, rdir, skip_remote_only_dirs)
		self.pd("---- Remote Dir Contents ---")
		self.pd(self._remote_dir_contents)

	def _compare(self, remotedir = None, localdir = None, skip_remote_only_dirs = False):
		if not localdir:
			localdir = '.'

		self.pv("Gathering local directory ...")
		self._gather_local_dir(localdir)
		self.pv("Done")
		self.pv("Gathering remote directory ...")
		self._gather_remote_dir(remotedir, skip_remote_only_dirs)
		self.pv("Done")
		self.pv("Comparing ...")
		# list merge, where Python shines
		commonsame = []
		commondiff = []
		localonly = []
		remoteonly = []
		# http://stackoverflow.com/questions/1319338/combining-two-lists-and-removing-duplicates-without-removing-duplicates-in-orig
		lps = self._local_dir_contents.allpath()
		rps = self._remote_dir_contents.allpath()
		dps = set(rps) - set(lps)
		allpath = lps + list(dps)
		for p in allpath:
			local = self._local_dir_contents.get(p)
			remote = self._remote_dir_contents.get(p)
			if local is None: # must be in the remote dir, since p is from allpath
				remoteonly.append((remote.type, p))
			elif remote is None:
				localonly.append((local.type, p))
			else: # all here
				same = False
				if local.type == 'D' and remote.type == 'D':
					type = 'D'
					same = True
				elif local.type == 'F' and remote.type == 'F':
					type = 'F'
					if local.extra['size'] == remote.extra['size'] and \
						(not self._verify or local.extra['md5'] == remote.extra['md5']):
						same = True
					else:
						same = False
				else:
					type = local.type + remote.type
					same = False

				if same:
					commonsame.append((type, p))
				else:
					commondiff.append((type, p))

		self.pv("Done")
		return commonsame, commondiff, localonly, remoteonly

	def compare(self, remotedir = None, localdir = None, skip_remote_only_dirs = False):
		''' Usage: compare [remotedir] [localdir] - \
compare the remote directory with the local directory
  remotedir - the remote directory at Baidu Yun (after app's directory). \
if not specified, it defaults to the root directory.
  localdir - the local directory, if not specified, it defaults to the current directory.
  skip_remote_only_dirs - skip remote-only sub-directories (faster if the remote \
directory is much larger than the local one). it defaults to False.
		'''
		same, diff, local, remote = self._compare(get_pcs_path(remotedir), localdir, str2bool(skip_remote_only_dirs))

		pr("==== Same files ===")
		for c in same:
			pr("{} - {}".format(c[0], c[1]))

		pr("==== Different files ===")
		for d in diff:
			pr("{} - {}".format(d[0], d[1]))

		pr("==== Local only ====")
		for l in local:
			pr("{} - {}".format(l[0], l[1]))

		pr("==== Remote only ====")
		for r in remote:
			pr("{} - {}".format(r[0], r[1]))

		pr("\nStatistics:")
		pr("--------------------------------")
		pr("Same: {}".format(len(same)))
		pr("Different: {}".format(len(diff)))
		pr("Local only: {}".format(len(local)))
		pr("Remote only: {}".format(len(remote)))

		self.result['same'] = same
		self.result['diff'] = diff
		self.result['local'] = local
		self.result['remote'] = remote

		return const.ENoError

	def _syncdown_diff_one(self, rpath, localdir, d):
		result = const.ENoError
		t = d[0]
		p = d[1]
		#lcpath = os.path.join(localdir, p) # local complete path
		lcpath = joinpath(localdir, p) # local complete path
		rcpath = rpath + '/' + p # remote complete path
		if t == 'DF':
			result = removedir(lcpath, self.verbose)
			subresult = self._downfile(rcpath, lcpath)
			if subresult != const.ENoError:
				result = subresult
		elif t == 'FD':
			result = removefile(lcpath, self.verbose)
			subresult = makedir(lcpath, verbose = self.verbose)
			if subresult != const.ENoError:
				result = subresult
		else: # " t == 'F' " must be true
			result = self._downfile(rcpath, lcpath)

		return result

	def _syncdown_remote_one(self, rpath, localdir, r):
		result = const.ENoError
		t = r[0]
		p = r[1]
		#lcpath = os.path.join(localdir, p) # local complete path
		lcpath = joinpath(localdir, p) # local complete path
		rcpath = rpath + '/' + p # remote complete path
		if t == 'F':
			subresult = self._downfile(rcpath, lcpath)
			if subresult != const.ENoError:
				result = subresult
		else: # " t == 'D' " must be true
			subresult = makedir(lcpath, verbose = self.verbose)
			if subresult != const.ENoError:
				result = subresult

		return result

	def _syncdown_delete_local(self, localdir, local):
		result = const.ENoError
		for l in local:
			# use os.path.isfile()/isdir() instead of l[0], because we need to check file/dir existence.
			# as we may have removed the parent dir previously during the iteration
			#p = os.path.join(localdir, l[1])
			p = joinpath(localdir, l[1])
			if os.path.isfile(p):
				subresult = removefile(p, self.verbose)
				if subresult != const.ENoError:
					result = subresult
			elif os.path.isdir(p):
				subresult = removedir(p, self.verbose)
				if subresult != const.ENoError:
					result = subresult

		return result

	def _syncdown_single(self, rpath, localdir, compare_result):
		result = const.ENoError
		same, diff, local, remote = compare_result
		# clear the way
		for d in diff:
			subresult = self._syncdown_diff_one(rpath, localdir, d)
			if subresult != const.ENoError:
				result = subresult
		for r in remote:
			subresult = self._syncdown_remote_one(rpath, localdir, r)
			if subresult != const.ENoError:
				result = subresult
		return result

	def _syncdown_multi(self, rpath, localdir, compare_result):
		result = const.ENoError
		same, diff, local, remote = compare_result
		# clear the way
		worker = partial(self._syncdown_diff_one, rpath, localdir)
		subresult = self._multi_process(worker, diff, "Sync down")
		if subresult != const.ENoError:
			result = subresult
		worker = partial(self._syncdown_remote_one, rpath, localdir)
		subresult = self._multi_process(worker, remote, "")
		if subresult != const.ENoError:
			result = subresult
		return result

	def syncdown(self, remotedir = '', localdir = '', deletelocal = False):
		''' Usage: syncdown [remotedir] [localdir] [deletelocal] - \
sync down from the remote directory to the local directory
  remotedir - the remote directory at Baidu Yun (after app's directory) to sync from. \
if not specified, it defaults to the root directory
  localdir - the local directory to sync to if not specified, it defaults to the current directory.
  deletelocal - delete local files that are not inside Baidu Yun directory, default is False
		'''
		result = const.ENoError
		rpath = get_pcs_path(remotedir)
		compare_result = self._compare(rpath, localdir, False)
		same, diff, local, remote = compare_result
		if Pool and self.processes > 1:
			subresult = self._syncdown_multi(rpath, localdir, compare_result)
			if subresult != const.ENoError:
				result = subresult
		else:
			subresult = self._syncdown_single(rpath, localdir, compare_result)
			if subresult != const.ENoError:
				result = subresult
		if str2bool(deletelocal):
			subresult = self._syncdown_delete_local(localdir, local)
			if subresult != const.ENoError:
				result = subresult

		if result == const.ENoError:
			self._remove_remote_on_success(rpath)
		return result

	def _syncup_diff_one(self, rpath, localdir, d):
		result = const.ENoError
		t = d[0] # type
		p = d[1] # path
		#lcpath = os.path.join(localdir, p) # local complete path
		lcpath = joinpath(localdir, p) # local complete path
		rcpath = rpath + '/' + p # remote complete path
		if self.shalloverwrite("Do you want to overwrite '{}' at Baidu Yun? [y/N]".format(p)):
			# this path is before get_pcs_path() since delete() expects so.
			#result = self.delete(rpartialdir + '/' + p)
			result = self._delete(rcpath)
#				self.pd("diff type: {}".format(t))
#				self._isrev = True
#				if t != 'F':
#					result = self.move(remotedir + '/' + p, remotedir + '/' + p + '.moved_by_bypy.' + time.strftime("%Y%m%d%H%M%S"))
#					self._isrev = False
			if t == 'F' or t == 'FD':
				subresult = self._upload_file(lcpath, rcpath)
				if subresult != const.ENoError:
					result = subresult
			else: # " t == 'DF' " must be true
				subresult = self._mkdir(rcpath)
				if subresult != const.ENoError and subresult != const.IEFileAlreadyExists:
					result = subresult
		else:
			pinfo("Uploading '{}' skipped".format(lcpath))

		return result

	def _syncup_local_one(self, rpath, localdir, l):
		result = const.ENoError
		t = l[0]
		p = l[1]
		#lcpath = os.path.join(localdir, p) # local complete path
		lcpath = joinpath(localdir, p) # local complete path
		rcpath = rpath + '/' + p # remote complete path
		self.pd("local type: {}".format(t))
		self._isrev = False
		if t == 'F':
			subresult = self._upload_file(lcpath, rcpath)
			if subresult != const.ENoError:
				result = subresult
		else: # " t == 'D' " must be true
			subresult = self._mkdir(rcpath)
			if subresult != const.ENoError and subresult != const.IEFileAlreadyExists:
				result = subresult

		return result

	def _syncup_delete_remote(self, rpath, remote):
		result = const.ENoError
		# i think the list is built top-down, so directories appearing later are either
		# children or another set of directories
		pp = '\\' # previous path, setting to '\\' make sure it won't be found in the first step
		for r in remote:
			#p = rpartialdir + '/' + r[1]
			p = rpath + '/' + r[1]
			if 0 != p.find(pp): # another path
				#subresult = self.delete(p)
				subresult = self._delete(p)
				if subresult != const.ENoError:
					result = subresult
			pp = p

		return result

	def _syncup_single(self, localdir, rpath, compare_result):
		result = const.ENoError
		same, diff, local, remote = compare_result
		# clear the way
		for d in diff:
			subresult = self._syncup_diff_one(rpath, localdir, d)
			if subresult != const.ENoError:
				result = subresult
		for l in local:
			subresult = self._syncup_local_one(rpath, localdir, l)
			if subresult != const.ENoError:
				result = subresult
		return result

	def _syncup_multi(self, localdir, rpath, compare_result):
		result = const.ENoError
		same, diff, local, remote = compare_result
		# clear the way
		worker = partial(self._syncup_diff_one, rpath, localdir)
		subresult = self._multi_process(worker, diff, "Sync up")
		if subresult != const.ENoError:
			result = subresult
		worker = partial(self._syncup_local_one, rpath, localdir)
		subresult = self._multi_process(worker, local, "")
		if subresult != const.ENoError:
			result = subresult
		return result

	def syncup(self, localdir = '', remotedir = '', deleteremote = False):
		''' Usage: syncup [localdir] [remotedir] [deleteremote] - \
sync up from the local directory to the remote directory
  localdir - the local directory to sync from if not specified, it defaults to the current directory.
  remotedir - the remote directory at Baidu Yun (after app's directory) to sync to. \
if not specified, it defaults to the root directory
  deleteremote - delete remote files that are not inside the local directory, default is False
		'''
		result = const.ENoError
		rpath = get_pcs_path(remotedir)
		#rpartialdir = remotedir.rstrip('/ ')
		compare_result = self._compare(rpath, localdir, True)
		same, diff, local, remote = compare_result
		if Pool and self.processes > 1:
			subresult = self._syncup_multi(localdir, rpath, compare_result)
			if subresult != const.ENoError:
				result = subresult
		else:
			subresult = self._syncup_single(localdir, rpath, compare_result)
			if subresult != const.ENoError:
				result = subresult
		if str2bool(deleteremote):
			subresult = self._syncup_delete_remote(rpath, remote)
			if subresult != const.ENoError:
				result = subresult
		if result == const.ENoError:
			self._remove_local_on_success(localdir)
		return result

	def dumpcache(self):
		''' Usage: dumpcache - display file hash cache'''
		if cached.cacheloaded:
			#pprint.pprint(cached.cache)
			MyPrettyPrinter().pprint(cached.cache)
			return const.ENoError
		else:
			perr("Cache not loaded.")
			return const.ECacheNotLoaded

	def cleancache(self):
		''' Usage: cleancache - remove invalid entries from hash cache file'''
		if os.path.exists(self._hashcachepath):
			try:
				# backup first
				backup = self._hashcachepath + '.lastclean'
				shutil.copy(self._hashcachepath, backup)
				self.pd("Hash Cache file '{}' backed up as '{}".format(
					self._hashcachepath, backup))
				cached.cleancache()
				return const.ENoError
			except Exception as ex:
				perr(formatex(ex))
				return const.EException
		else:
			return const.EFileNotFound

	def _cdl_act(self, r, args):
		try:
			pr(pprint.pformat(r.json()))
			return const.ENoError
		except:
			pr(pprint.pformat({ 'text': rb(r.text) }))
			return const.IETaskNotFound

	def _prepare_cdl_add(self, source_url, rpath, timeout):
		pr("Adding cloud download task:")
		pr("{} =cdl=> {}".format(source_url, rpath))
		pars = {
			'method': 'add_task',
			'source_url': source_url,
			'save_path': rpath,
			'timeout': 3600 }
		return pars

	def _cdl_add(self, source_url, rpath, timeout):
		pars = self._prepare_cdl_add(source_url, rpath, timeout)
		return self._post(pcsurl + 'services/cloud_dl', pars, self._cdl_act)

	def _get_cdl_dest(self, source_url, save_path):
		rpath = get_pcs_path(save_path)
		# download to /apps/bypy root
		if rpath == const.AppPcsPath \
			or (const.ENoError == self._get_file_info(rpath) \
				and self._remote_json['isdir']):
			filename = source_url.split('/')[-1]
			rpath += '/' + filename
		return rpath

	def cdl_add(self, source_url, save_path = '/', timeout = 3600):
		''' Usage: cdl_add <source_url> [save_path] [timeout] - add an offline (cloud) download task
  source_url - the URL to download file from.
  save_path - path on PCS to save file to. default is to save to root directory '/'.
  timeout - timeout in seconds. default is 3600 seconds.
		'''
		rpath = self._get_cdl_dest(source_url, save_path)
		return self._cdl_add(source_url, rpath, timeout)

	def _get_cdl_query_pars(self, task_ids, op_type):
		pars = {
			'method': 'query_task',
			'task_ids': task_ids,
			'op_type': op_type}
		return pars

	def _cdl_query(self, task_ids, op_type):
		pars =  self._get_cdl_query_pars(task_ids, op_type)
		return self._post(pcsurl + 'services/cloud_dl', pars, self._cdl_act)

	def cdl_query(self, task_ids, op_type = 1):
		''' Usage: cdl_query <task_ids>  - query existing offline (cloud) download tasks
  task_ids - task ids seperated by comma (,).
  op_type - 0 for task info; 1 for progress info. default is 1
		'''
		return self._cdl_query(task_ids, op_type)

	def _cdl_mon_act(self, r, args):
		try:
			task_id, start_time, done = args
			j = r.json()
			ti = j['task_info'][str(task_id)]
			if ('file_size' not in ti) or ('finished_size' not in ti):
				done[0] = True
				pr(j)
			else:
				total = int(ti['file_size'])
				finish = int(ti['finished_size'])
				done[0] = (total != 0 and (total == finish))
				pprgr(finish, total, start_time)
				if done[0]:
					pr(pprint.pformat(j))
			return const.ENoError
		except Exception as ex:
			perr("Exception while monitoring offline (cloud) download task:\n{}".format(formatex(ex)))
			perr("Baidu returned:\n{}".format(rb(r.text)))
			return const.EInvalidJson

	def _cdl_addmon_act(self, r, args):
		try:
			args[0] = r.json()
			pr(pprint.pformat(args[0]))
			return const.ENoError
		except Exception as ex:
			perr("Exception while adding offline (cloud) download task:\n{}".format(formatex(ex)))
			perr("Baidu returned:\n{}".format(rb(r.text)))
			return const.EInvalidJson

	def _cdl_sighandler(self, signum, frame):
		pr("Cancelling offline (cloud) download task: {}".format(self._cdl_task_id))
		result = self._cdl_cancel(self._cdl_task_id)
		pr("Result: {}".format(result))
		sys.exit(const.EAbort)

	def _cdl_addmon(self, source_url, rpath, timeout = 3600):
		pars = self._prepare_cdl_add(source_url, rpath, timeout)
		jc = [{}] # out param
		result = self._post(pcsurl + 'services/cloud_dl',
			pars, self._cdl_addmon_act, jc)
		if result == const.ENoError:
			if not 'task_id' in jc[0]:
				return const.EInvalidJson
			task_id = jc[0]['task_id']
			pars = self._get_cdl_query_pars(task_id, 1)
			start_time = time.time()
			done = [ False ] # out param
			# cancel task on Ctrl-C
			pr("Press Ctrl-C to cancel the download task")
			self._cdl_task_id = task_id
			setsighandler(signal.SIGINT, self._cdl_sighandler)
			setsighandler(signal.SIGHUP, self._cdl_sighandler)
			try:
				while True:
					result = self._post(
						pcsurl + 'services/cloud_dl', pars, self._cdl_mon_act,
						(task_id, start_time, done))
					if result == const.ENoError:
						if done[0] == True:
							return const.ENoError
					else:
						return result
					time.sleep(5)
			except KeyboardInterrupt:
				pr("Canceling offline (cloud) downloa task: {}".format(task_id))
				self._cdl_cancel(task_id)
				return const.EAbort
		else:
			return result

	def cdl_addmon(self, source_url, save_path = '/', timeout = 3600):
		''' Usage: cdl_addmon <source_url> [save_path] [timeout] - add an offline (cloud) download task and monitor the download progress
  source_url - the URL to download file from.
  save_path - path on PCS to save file to. default is to save to root directory '/'.
  timeout - timeout in seconds. default is 3600 seconds.
		'''
		rpath = self._get_cdl_dest(source_url, save_path)
		return self._cdl_addmon(source_url, rpath, timeout)

	def _cdl_list(self):
		pars = {
			'method': 'list_task' }
		return self._post(pcsurl + 'services/cloud_dl', pars, self._cdl_act)

	def cdl_list(self):
		''' Usage: cdl_list - list offline (cloud) download tasks
		'''
		return self._cdl_list()

	def _cdl_cancel(self, task_id):
		pars = {
			'method': 'cancel_task',
			'task_id': task_id }
		return self._post(pcsurl + 'services/cloud_dl', pars, self._cdl_act)

	def cdl_cancel(self, task_id):
		''' Usage: cdl_cancel <task_id>  - cancel an offline (cloud) download task
  task_id - id of the task to be canceled.
		'''
		return self._cdl_cancel(task_id)

	def _get_accept_cmd(self, rpath):
		md5str = self._current_file_md5
		slicemd5str =  self._current_file_slice_md5
		crcstr = hex(self._current_file_crc32)
		remotepath = rpath[const.AppPcsPathLen:]
		if len(remotepath) == 0:
			remotepath = 'PATH_NAME_MISSING'
		cmd = "bypy accept {} {} {} {} {}".format(
			remotepath, self._current_file_size, md5str, slicemd5str, crcstr)
		return cmd

	def _share_local_file(self, lpath, rpath, fast):
		filesize = getfilesize(lpath)
		if filesize < const.MinRapidUploadFileSize:
			perr("File size ({}) of '{}' is too small (must be greater or equal than {}) to be shared".format(
				human_size(filesize), lpath, human_size(const.MinRapidUploadFileSize)))
			return const.EParameter

		if fast:
			self._get_hashes_for_rapidupload(lpath, setlocalfile = True)
			pr(self._get_accept_cmd(rpath))
			return const.ENoError

		ulrpath = const.RemoteTempDir + '/' + posixpath.basename(lpath)
		result = self._upload_file(lpath, ulrpath)
		if result != const.ENoError:
			perr("Unable to share as uploading failed")
			return result

		if not self._rapiduploaded:
			i = 0
			while i < const.ShareRapidUploadRetries:
				i += 1
				result = self._rapidupload_file(lpath, ulrpath, setlocalfile = True)
				if result == const.ENoError: # or result == IEMD5NotFound: # retrying if MD5 not found _may_ make the file available?
					break
				else:
					self.pd("Retrying #{} for sharing '{}'".format(i, lpath))
					time.sleep(1)

		if result == const.ENoError:
			pr(self._get_accept_cmd(rpath))
			return const.ENoError
		elif result == const.IEMD5NotFound:
			pr("# Sharing (RapidUpload) not possible for '{}', error: {}".format(lpath, result))
			return result
		else:
			pr("# Error sharing '{}', error: {}".format(lpath, result))
			return result

	def _share_local_dir(self, lpath, rpath, fast):
		result = const.ENoError
		for walk in self._walk_normal_file(lpath):
			(dirpath, dirnames, filenames) = walk
			for filename in filenames:
				rpart = os.path.relpath(dirpath, lpath)
				if rpart == '.':
					rpart = ''
				subr = self._share_local_file(
					joinpath(dirpath, filename),
					posixpath.join(rpath, rpart, filename),
					fast)
				if subr != const.ENoError:
					result = subr
		return result

	# assuming the caller asks once only
	def _ok_to_use_remote_temp_dir(self):
		if const.SettingKey_OverwriteRemoteTempDir in self._setting and \
		self._setting[const.SettingKey_OverwriteRemoteTempDir]:
			return True

		# need to check existence of the remote temp dir
		fir = self._get_file_info(const.RemoteTempDir)
		if fir == const.ENoError: # path exists
			msg = '''
In order to use this functionality, we need to use a temporary directory '{}' \
at Baidu Cloud Storage. However, this remote path exists already.
Is it OK to empty this directory? (Unless coincidentally you uploaded files there, \
it's probably safe to allow this)
y/N/a (yes/NO/always)?
'''.format(const.RemoteTempDir).strip()
			ans = ask(msg).lower()
			if len(ans) >= 0:
				a = ans[0]
				if a == 'y':
					return True
				elif a == 'a':
					# persist
					self._setting[const.SettingKey_OverwriteRemoteTempDir] = True
					self.savesetting()
					return True
				else:
					return False
			else:
				return False
		elif fir == const.EFileNotFound:
			return True
		elif fir == const.ERequestFailed:
			perr("Request to get info of the remote temp dir '{}' failed, can't continue.".format(const.RemoteTempDir))
			return False
		else:
			assert 0 == "Future work handling (more return code) needed"
			return False

	def _share_local(self, lpath, rpath, fast):
		if not os.path.exists(lpath):
			perr("Local path '{}' does not exist.".format(lpath))
			return const.EParameter

		if not self._ok_to_use_remote_temp_dir():
			perr("Can't continue unless you allow the program to use the remote temporary directory '{}'".format(const.RemoteTempDir))
			return const.EUserRejected

		if fast:
			pr("# fast (unverified) sharing, no network I/O needed (for local sharing), but the other person may not be able to accept some of your files")
		if os.path.isfile(lpath):
			# keep the same file name
			lname = os.path.basename(lpath)
			rfile = joinpath(rpath, lname, '/')
			return self._share_local_file(lpath, rfile, fast)
		elif os.path.isdir(lpath):
			return self._share_local_dir(lpath, rpath, fast)
		else:
			perr("Local path '{}' is not a file or directory.".format(lpath))
			return const.EParameter

	def _share_remote_file(self, tmpdir, rpath, srpath, fast):
		rdir, rfile = posixpath.split(rpath)
		lpath = joinpath(tmpdir, rfile)
		subr = self._downfile(rpath, lpath)
		if subr != const.ENoError:
			perr("Fatal: Error {} while downloading remote file '{}'".format(subr, rpath))
			return subr

		return self._share_local_file(lpath, srpath, fast)

	def _proceed_share_remote(self, walkresult, rpath, dirjs, filejs, args):
		remoterootlen, tmpdir, srpath, fast = args
		result = walkresult
		for filej in filejs:
			rfile = filej['path']
			subr = self._share_remote_file(tmpdir, rfile, joinpath(srpath, rfile[remoterootlen:], sep = '/'), fast)
			if subr != const.ENoError:
				result = subr
		return result

	def _share_remote_dir(self, tmpdir, rpath, srpath, fast):
		return self._walk_proceed_remote_dir(rpath, self._proceed_share_remote, (len(rpath), tmpdir, srpath, fast))

	def _share_remote(self, tmpdir, rpath, srpath, fast): # srpath - share remote path (full path)
		subr = self._get_file_info(rpath)
		if const.ENoError == subr:
			if 'isdir' in self._remote_json:
				if self._remote_json['isdir']:
					return self._share_remote_dir(tmpdir, rpath, srpath, fast)
				else:
					return self._share_remote_file(tmpdir, rpath, srpath, fast)
			else:
				perr("Malformed path info JSON '{}' returned".format(self._remote_json))
				return const.EFatal
		elif const.EFileNotFound == subr:
			perr("Remote path '{}' does not exist".format(rpath))
			return subr
		else:
			perr("Error {} while getting info for remote path '{}'".format(subr, rpath))
			return subr

	def share(self, path = '.', sharepath = '/', islocal = True, fast = False):
		islocal = str2bool(islocal)
		fast = str2bool(fast)
		if islocal:
			lpath = path
			rpath = get_pcs_path(sharepath)
			result = self._share_local(lpath, rpath, fast)
			if not fast:
				# not critical
				self._delete(const.RemoteTempDir)
			return result
		else:
			rpath = get_pcs_path(path)
			srpath = get_pcs_path(sharepath)
			tmpdir = tempfile.mkdtemp(prefix = 'bypy_')
			self.pd("Using local temporary directory '{}' for sharing".format(tmpdir))
			try:
				result = self._share_remote(tmpdir, rpath, srpath, fast)
			except Exception as ex:
				result = const.EFatal
				perr("Exception while sharing remote path '{}'.\n{}".format(
					rpath, formatex(ex)))
			finally:
				removedir(tmpdir)
			return result

	def _accept(self, rpath, size, md5str, slicemd5str, crcstr):
		# we have no file to verify against
		verify = self._verify
		self._verify = False
		result = self._rapidupload_file_post(rpath, size, md5str, slicemd5str, crcstr)
		self._verify = verify
		if result == const.ENoError:
			self.pv("Accepted: {}".format(rpath))
		else:
			perr("Unable to accept: {}, Error: {}".format(rpath, result))
		return result

	def accept(self, remotepath, size, md5str, slicemd5str, crcstr):
		rpath = get_pcs_path(remotepath)
		return self._accept(rpath, size, md5str, slicemd5str, crcstr)

	def _locate_fastest_upload_server_act(self, r, args):
		global cpcsurl
		j = r.json()
		cpcsurl = j['host']
		return const.ENoError

	# public Baidu API, no token needed, and not much useful. #127
	def _locate_fastest_upload_server(self):
		url = const.PcsUrl + 'file'
		pars = { 'method': 'locateupload'}
		return self._get(url, pars, self._locate_fastest_upload_server_act, addtoken = False)

	def _list_pcs_hosts_act(self, r, args):
		j = r.json()
		pr(pprint.pformat(j))
		return const.ENoError

	def _list_pcs_hosts(self):
		url = const.PcsUrl + 'manage'
		pars = { 'method': 'listhost'}
		return self._get(url, pars, self._list_pcs_hosts_act, addtoken = False)

	def _test_speed(self, url, repeats = 1):
		self.pd("Speed testing (GET) ({}x): {}".format(repeats, url))
		start = time.time()
		for i in range(repeats):
			requests.get(url)
		stop = time.time()
		duration = stop - start
		self.pd("Time taken: {:.3f}s".format(duration))
		return duration

	def _select_fastest_mirror_act(self, r, args):
		pr("Selecting fastest mirror")
		j = r.json()
		self.pv("List of PCS mirrors:\n" + pprint.pformat(j))
		if 'path' not in j:
			return const.EInvalidJson
		pcspath = j['path']
		if 'list' not in j:
			return const.EInvalidJson
		hosts = j['list']
		if len(hosts) < 1:
			return const.EInvalidJson
		selected_host = hosts[0]['host']
		shortest_time = float('inf')
		for h in hosts:
			host = h['host']
			try:
				time_taken = self._test_speed('https://' + host + pcspath, 3)
				if time_taken < shortest_time:
					selected_host = host
					shortest_time = time_taken
			except Exception as ex:
				perr("Error while testing speed of {}: {}, skipping.".format(host, formatex(ex)))
				continue
		pr("Selected mirror: " + selected_host)
		self.setpcsurl(selected_host)
		return const.ENoError

	def _select_fastest_mirror(self):
		url = const.PcsUrl + 'manage'
		pars = { 'method': 'listhost'}
		return self._get(url, pars, self._select_fastest_mirror_act, addtoken = False)

def sighandler(signum, frame):
	pr("Signal {} received, Abort".format(signum))
	sys.exit(const.EAbort)

# http://www.gnu.org/software/libc/manual/html_node/Basic-Signal-Handling.html
def setsighandler(signum, handler):
	oldhandler = signal.signal(signum, handler)
	if oldhandler == signal.SIG_IGN:
		signal.signal(signum, signal.SIG_IGN)
	return oldhandler

def setuphandlers():
	if iswindows():
		# setsighandler(signal.CTRL_C_EVENT, sighandler)
		# setsighandler(signal.CTRL_BREAK_EVENT, sighandler)
		# bug, see: http://bugs.python.org/issue9524
		pass
	else:
		setsighandler(signal.SIGBUS, sighandler)
		setsighandler(signal.SIGHUP, sighandler)
		# https://stackoverflow.com/questions/108183/how-to-prevent-sigpipes-or-handle-them-properly
		setsighandler(signal.SIGPIPE, signal.SIG_IGN)
		setsighandler(signal.SIGQUIT, sighandler)
		setsighandler(signal.SIGSYS, sighandler)
	setsighandler(signal.SIGABRT, sighandler)
	setsighandler(signal.SIGFPE, sighandler)
	setsighandler(signal.SIGILL, sighandler)
	setsighandler(signal.SIGINT, sighandler)
	setsighandler(signal.SIGSEGV, sighandler)
	setsighandler(signal.SIGTERM, sighandler)

def getparser():
	#name = os.path.basename(sys.argv[0])
	version = "v%s" % const.__version__
	version_message = '%%(prog)s %s' % (version)
	desc = "{} - {}".format(version_message, const.__desc__)

	# setup argument parser
	epilog = "Commands:\n"
	summary = []
	for k, v in ByPy.__dict__.items():
		if callable(v):
			if v.__doc__:
				help = v.__doc__.strip()
				pos = help.find(const.HelpMarker)
				if pos != -1:
					pos_body = pos + len(const.HelpMarker)
					helpbody = help[pos_body:]
					helpline = helpbody.split('\n')[0].strip() + '\n'
					if helpline.find('help') == 0:
						summary.insert(0, helpline)
					else:
						summary.append(helpline)
					#commands.append(v.__name__) # append command name

	remaining = summary[1:]
	remaining.sort()
	summary = [summary[0]] + remaining
	epilog += ''.join(summary)

	parser = ArgumentParser(
		prog = const.__title__,
		description=desc,
		formatter_class=RawDescriptionHelpFormatter, epilog=epilog)

	# help, version, program information etc
	parser.add_argument('-V', '--version', action='version', version=version_message)

	# debug, logging
	parser.add_argument("-d", "--debug",
		dest="debug", action="count", default=0,
		help="set debugging level (-dd to increase debugging level, -ddd to enable HTPP traffic debugging as well (very talkative)) [default: %(default)s]")
	parser.add_argument("-v", "--verbose", dest="verbose", default=0, action="count", help="set verbosity level [default: %(default)s]")

	# program tunning, configration (those will be passed to class ByPy)
	parser.add_argument("-r", "--retry",
		dest="retry", default=5, type=int,
		help="number of retry attempts on network error [default: %(default)i times]")
	parser.add_argument("-q", "--quit-when-fail",
		dest="quit", action="store_true",
		help="quit when maximum number of retry failed [default: %(default)s]")
	parser.add_argument("-t", "--timeout",
		dest="timeout", default=const.DefaultTimeOutInSeconds, type=float,
		help="network timeout in seconds [default: %(default)s]")
	parser.add_argument("-s", "--slice",
		dest="slice", default=const.DefaultSliceSize,
		help="size of file upload slice (can use '1024', '2k', '3MB', etc) [default: {} MB]".format(const.DefaultSliceInMB))
	parser.add_argument("--chunk",
		dest="chunk", default=const.DefaultDlChunkSize,
		help="size of file download chunk (can use '1024', '2k', '3MB', etc) [default: {} MB]".format(const.DefaultDlChunkSize // const.OneM))
	parser.add_argument("-e", "--verify",
		dest="verify", action="store_true",
		help="verify upload / download [default : %(default)s]")
	parser.add_argument("-f", "--force-hash",
		dest="forcehash", action="store_true",
		help="force file MD5 / CRC32 calculation instead of using cached value")
	parser.add_argument("--no-resume-download",
		dest="resumedl", action="store_false",
		help="resume instead of restarting when downloading if local file already exists [default: %(default)s]")
	parser.add_argument("--include-regex",
		dest="incregex", default='',
		help="regular expression of files to include. if not specified (default), everything is included. for download, the regex applies to the remote files; for upload, the regex applies to the local files. to exclude files, think about your regex, some tips here: https://stackoverflow.com/questions/406230/regular-expression-to-match-string-not-containing-a-word [default: %(default)s]")
	parser.add_argument("--on-dup",
		dest="ondup", default='overwrite',
		help="what to do when the same file / folder exists in the destination: 'overwrite', 'skip', 'prompt' [default: %(default)s]")
	parser.add_argument("--no-symlink",
		dest="followlink", action="store_false",
		help="DON'T follow symbol links when uploading / syncing up")
	parser.add_argument(const.DisableSslCheckOption,
		dest="checkssl", action="store_false",
		help="DON'T verify host SSL cerificate")
	parser.add_argument(const.CaCertsOption,
		dest="cacerts", default=None,
		help="Specify the path for CA Bundle [default: %(default)s]")
	parser.add_argument("--mirror",
		dest="mirror", default='',
		help="Specify the PCS mirror (e.g. bj.baidupcs.com. Open 'https://pcs.baidu.com/rest/2.0/pcs/manage?method=listhost' to get the list) to use. [default: " + const.PcsDomain + "]")
	parser.add_argument("--select-fastest-mirror",
		dest="selectmirror", action="store_true",
		help="Let the program run some tests and select the fastest PCS mirror it detectes. [default: %(default)s]")
	parser.add_argument("--rapid-upload-only",
		dest="rapiduploadonly", action="store_true",
		help="only upload large files that can be rapidly uploaded")
	parser.add_argument("--resume-download-revert-back",
		dest="resumedl_revertcount", default=const.DefaultResumeDlRevertCount,
		type=int, metavar='RCOUNT',
		help="Revert back at least %(metavar)s download chunk(s) and align to chunk boundary when resuming the download. A negative value means NO reverts. [default: %(default)s]")
	parser.add_argument("--move",
		dest="deletesource", action="store_true",
		help="Delete source files/directories after download/upload/syncdown/syncup is successful (This will force verification of the files). [default: %(default)s]")
	if Pool:
		parser.add_argument(const.MultiprocessOption,
			dest="processes", default=const.DefaultProcessCount, type=int,
			help="Number of parallel processes. (Only applies to dir sync/dl/ul). [default: %(default)s]")

	# support aria2c
	parser.add_argument("--downloader",
		dest="downloader", default="",
		help="downloader to use (use python if not specified). valid values: {} [default: %(default)s]".format(const.Downloaders))
	parser.add_argument("--downloader-arguments",
		dest="downloader_args", default="",
		help=("arguments for the downloader:\n"
			"normally, the string is the arguments to be passed to the downloader. "
			"however, when it begins with '{0}', it will be treated as the name of file, "
			"whose contents will be used as the downloader arguments "
			"(example: when specifying '{0}args.txt', file contents of 'args.txt' will be "
			"used as the downloader arguments, not the string '@args.txt' itself).\n"
			"you can also use environment variable '{1}' to specify the downloader arguments "
			"(the environment variable has lower priority compared to this argument).\n"
			"default values: {2}"
			).format(
				const.DownloaderArgsIsFilePrefix,
				const.DownloaderArgsEnvKey,
				const.DownloaderDefaultArgs))

	# expose this to provide a primitive multi-user support
	parser.add_argument("--config-dir", dest="configdir", default=const.ConfigDir, help="specify the config path [default: %(default)s]")

	# action
	parser.add_argument(const.CleanOptionShort, const.CleanOptionLong,
		dest="clean", action="count", default=0,
		help="remove the token file (need re-auth) and upload progress file, "
			"-cc: clean hash cache file as well")

	# the MAIN parameter - what command to perform
	parser.add_argument("command", nargs='*', help = "operations (quota, list, etc)")

	return parser

def clean_prog_files(cleanlevel, verbose, configdir = const.ConfigDir):
	tokenpath = os.path.join(configdir, const.TokenFileName)
	result = removefile(tokenpath, verbose)
	if result == const.ENoError:
		pr("Token file '{}' removed. You need to re-authorize "
		   "the application upon next run".format(tokenpath))
	else:
		perr("Failed to remove the token file '{}'".format(tokenpath))
		perr("You need to remove it manually")

	prgrpath = os.path.join(configdir, const.ProgressFileName)
	subresult = removefile(prgrpath, verbose)
	if result == const.ENoError:
		pr("Progress file '{}' removed, "
			"any upload will be started from the beginning.".format(prgrpath))
	else:
		perr("Failed to remove the progress file '{}'".format(prgrpath))
		result = subresult

	if cleanlevel >= 2:
		cachepath = os.path.join(configdir, const.HashCacheFileName)
		subresult = os.remove(cachepath)
		if subresult == const.ENoError:
			pr("Hash Cache File '{}' removed.".format(cachepath))
		else:
			perr("Failed to remove the Hash Cache File '{}'".format(cachepath))
			perr("You need to remove it manually")
			result = subresult

	return result

def printBaiduBanner():
	banner ='''
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Baidu PCS currently gives totally WRONG MD5 hash for remote files, and I AM NOT GOING TO ADAPT TO IT!
So if it doesn't work, install version '1.6.10' using the following command:
pip install bypy==1.6.10
Version 1.6.10 has a workaround for this.
### Seeking maintainer for this `bypy` project, if you are interested, please create an issue at github, thanks. ###
----------------------------------------------------------------
百度云盘返回的MD5全错了，鬼知道他们什么时候会改回来！现在代码是按照MD5值是正确的前提来写的，不会改。
所以你发现比较功能等不能正确工作时，请安装 1.6.10 版本：
pip install bypy==1.6.10
1.6.10版有个临时解决方案。
### 寻求这个`bypy`项目维护者，有意的话请去github上建issue，谢谢。###
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
'''
	print(banner)

def main(argv=None): # IGNORE:C0111
	''' Main Entry '''
	# printBaiduBanner()

	by = None
	reqres = check_requirements()
	if reqres == CheckResult.Error:
		perr("Requirement checking failed")
		sys.exit(const.EFatal)

	result = const.ENoError
	if argv is None:
		argv = sys.argv
	else:
		sys.argv.extend(argv)

	setuphandlers()

	parser = getparser()
	args = parser.parse_args()
	dl_args = ''
	if not args.downloader_args:
		if const.DownloaderArgsEnvKey in os.environ:
			dl_args = os.environ[const.DownloaderArgsEnvKey]
	else:
		prefixlen = len(const.DownloaderArgsIsFilePrefix)
		if args.downloader_args[:prefixlen] == const.DownloaderArgsIsFilePrefix: # file
			with io.open(args.downloader_args[prefixlen:], 'r', encoding = 'utf-8') as f:
				dl_args = f.read().strip()
		else:
			dl_args = args.downloader_args

	# house-keeping reminder
	# TODO: may need to move into ByPy for customized config dir
	if os.path.exists(const.HashCachePath):
		cachesize = getfilesize(const.HashCachePath)
		if cachesize > 10 * const.OneM or cachesize == -1:
			pwarn((
				"*** WARNING ***\n"
				"Hash Cache file '{0}' is very large ({1}).\n"
				"This may affect program's performance (high memory consumption).\n"
				"You can first try to run 'bypy.py cleancache' to slim the file.\n"
				"But if the file size won't reduce (this warning persists),"
				" you may consider deleting / moving the Hash Cache file '{0}'\n"
				"*** WARNING ***\n\n\n").format(const.HashCachePath, human_size(cachesize)))

	# check for situations that require no ByPy object creation first
	if args.clean >= 1:
		return clean_prog_files(args.clean, args.verbose, args.configdir)

	# some arguments need some processing
	try:
		slice_size = interpret_size(args.slice)
	except (ValueError, KeyError):
		perr("Error: Invalid slice size specified '{}'".format(args.slice))
		return const.EArgument

	try:
		chunk_size = interpret_size(args.chunk)
	except (ValueError, KeyError):
		perr("Error: Invalid slice size specified '{}'".format(args.slice))
		return const.EArgument

	if len(args.command) <= 0 or \
		(len(args.command) == 1 and args.command[0].lower() == 'help'):
		parser.print_help()
		return const.EArgument
	elif len(args.command) == 2 and args.command[0].lower() == 'help':
		ByPy.help(args.command[1])
		return const.EArgument
	elif args.command[0] in ByPy.__dict__: # dir(ByPy), dir(by)
		#timeout = args.timeout or None

		cached.usecache = not args.forcehash
		bypyopt = {
			'slice_size': slice_size,
			'dl_chunk_size': chunk_size,
			'verify': args.verify,
			'retry': args.retry,
			'timeout': args.timeout,
			'quit_when_fail': args.quit,
			'resumedownload': args.resumedl,
			'incregex': args.incregex,
			'ondup': args.ondup,
			'followlink': args.followlink,
			'checkssl': args.checkssl,
			'cacerts': args.cacerts,
			'rapiduploadonly': args.rapiduploadonly,
			'mirror': args.mirror,
			'selectmirror': args.selectmirror,
			'configdir': args.configdir,
			'resumedl_revertcount': args.resumedl_revertcount,
			'deletesource': args.deletesource,
			'downloader': args.downloader,
			'downloader_args': dl_args,
			'verbose': args.verbose,
			'debug': args.debug}
		if Pool:
			bypyopt['processes'] = args.processes

		# we construct a ByPy object here.
		# if you want to try PanAPI, simply replace ByPy with PanAPI, and all the bduss related function _should_ work
		# I didn't use PanAPI here as I have never tried out those functions inside
		by = ByPy(**bypyopt)
		uargs = []
		for arg in args.command[1:]:
			if sys.version_info[0] < 3:
				uargs.append(unicode(arg, gvar.SystemEncoding))
			else:
				uargs.append(arg)
		result = getattr(by, args.command[0])(*uargs)
		if result != const.ENoError:
			errmsg = '-' * 64 + "\nError {}{}".format(result, ': ' + const.ErrorExplanations[result] if result in const.ErrorExplanations else '')
			perr(errmsg)
	else:
		perr("Error: Command '{}' not available.".format(args.command[0]))
		parser.print_help()
		return const.EParameter

	# just let it spew exceptions so that we have the stack trace on death
	# except KeyboardInterrupt:
	# 	# handle keyboard interrupt
	# 	pr("KeyboardInterrupt")
	# 	pr("Abort")
	# except Exception as ex:
	# 	# NOTE: Capturing the exeption as 'ex' seems to matter, otherwise this:
	# 	# except Exception:
	# 	# will sometimes give exception ...
	# 	perr("Exception occurred:\n{}".format(formatex(ex)))
	# 	pr("Abort")
	#	raise

if __name__ == "__main__":
	main()

# vim: tabstop=4 noexpandtab shiftwidth=4 softtabstop=4 ff=unix fileencoding=utf-8
