From 607e035761250d2fbdd44fd07f7e13e043383b92 Mon Sep 17 00:00:00 2001 From: Jan Max Meyer Date: Wed, 22 Mar 2017 16:53:49 +0100 Subject: [PATCH 1/5] Temporary commiting current working files. --- modules/user.py | 21 +++++++++++---------- render/html/user.py | 26 +++++++++++++++++++++----- render/json/user.py | 7 +++++-- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/modules/user.py b/modules/user.py index af2fc50..55162cd 100644 --- a/modules/user.py +++ b/modules/user.py @@ -326,7 +326,7 @@ def startProcessing(self, userKey): "timestamp": time(), "failures": 0} session.current.markChanged() - return self.userModule.render.loginSucceeded(msg="X-VIUR-2FACTOR-TimeBasedOTP") + return self.userModule.render.loginSecondFactor("X-VIUR-2FACTOR-TimeBasedOTP") return None @@ -363,13 +363,13 @@ def asBytes(valIn): @exposed @forceSSL - def otp(self, otptoken = None, skey = None, *args, **kwargs ): + def otp(self, otptoken = None, skey = None, *args, **kwargs): token = session.current.get("_otp_user") if not token: raise errors.Forbidden() if otptoken is None: - self.userModule.render.edit(self.otpSkel()) + self.userModule.render.edit(self.otpSkel(), **kwargs) if not securitykey.validate(skey): raise errors.PreconditionFailed() @@ -381,12 +381,12 @@ def otp(self, otptoken = None, skey = None, *args, **kwargs ): try: otptoken = int(otptoken) except: - # We got a non-numeric token - this cant be correct - self.userModule.render.edit(self.otpSkel()) + # We got a non-numeric token - this can't be correct + self.userModule.render.edit(self.otpSkel(), **kwargs) - logging.debug(otptoken) - logging.debug(validTokens) - logging.debug(otptoken in validTokens) + #logging.debug(otptoken) + #logging.debug(validTokens) + #logging.debug(otptoken in validTokens) if otptoken in validTokens: userKey = session.current["_otp_user"]["uid"] @@ -407,7 +407,7 @@ def otp(self, otptoken = None, skey = None, *args, **kwargs ): session.current["_otp_user"] = token session.current.markChanged() - return self.userModule.render.edit(self.otpSkel(), loginFailed=True) + return self.userModule.render.edit(self.otpSkel(), loginFailed=True, **kwargs) def updateTimeDrift(self, userKey, idx): """ @@ -570,7 +570,7 @@ def authenticateUser(self, userKey, **kwargs): del oldSession session.current["user"] = {} - + for key in ["name", "status", "access"]: try: session.current["user"][key] = res[key] @@ -626,6 +626,7 @@ def view(self, key, *args, **kwargs): """ Allow a special key "self" to reference always the current user """ + if key == "self": user = self.getCurrentUser() if user: diff --git a/render/html/user.py b/render/html/user.py index 3941e25..1666f9b 100644 --- a/render/html/user.py +++ b/render/html/user.py @@ -6,22 +6,38 @@ class Render( default.Render ): #Render user-data to xml loginTemplate = "user_login" - logoutSuccessTemplate = "user_logout_success" + loginSecondFactorTemplate = "user_login_secondfactor" loginSuccessTemplate = "user_login_success" + logoutSuccessTemplate = "user_logout_success" verifySuccessTemplate = "user_verify_success" verifyFailedTemplate = "user_verify_failed" passwdRecoverInfoTemplate = "user_passwdrecover_info" - def login( self, skel, tpl=None, **kwargs ): - return( self.edit( skel, tpl=(tpl or self.loginTemplate), **kwargs ) ) + def login(self, skel, tpl=None, **kwargs): + if "loginTemplate" in dir(self.parent): + tpl = tpl or self.parent.loginTemplate + else: + tpl = tpl or self.loginTemplate - def loginSucceeded( self, tpl=None, **kwargs ): + return self.edit(skel, tpl=tpl, **kwargs) + + def loginSecondFactor(self, factor, tpl=None, **kwargs): + if "loginSecondFactorTemplate" in dir(self.parent): + tpl = tpl or self.parent.loginSecondFactorTemplate + else: + tpl = tpl or self.loginSecondFactorTemplate + + template= self.getEnv().get_template(self.getTemplateFileName(tpl)) + return template.render(factor=factor, **kwargs) + + def loginSuccess(self, tpl=None, **kwargs): if "loginSuccessTemplate" in dir( self.parent ): tpl = tpl or self.parent.loginSuccessTemplate else: tpl = tpl or self.loginSuccessTemplate + template= self.getEnv().get_template( self.getTemplateFileName( tpl ) ) - return( template.render( **kwargs ) ) + return template.render(**kwargs) def logoutSuccess(self, tpl=None, **kwargs ): if "logoutSuccessTemplate" in dir( self.parent ): diff --git a/render/json/user.py b/render/json/user.py index 12ac59d..62be629 100644 --- a/render/json/user.py +++ b/render/json/user.py @@ -11,8 +11,11 @@ def login(self, skel, **kwargs): return self.edit(skel, **kwargs) - def loginSucceeded(self, msg = "OKAY", **kwargs): - return json.dumps(msg) + def loginSecondFactor(self, factor, **kwargs): + return json.dumps(factor) + + def loginSucceeded(self, **kwargs): + return json.dumps("OKAY") def logoutSuccess(self, **kwargs): return json.dumps("OKAY") From ae981b39d1cbaf279a9865f7708b8099ab72657c Mon Sep 17 00:00:00 2001 From: Jan Max Meyer Date: Wed, 22 Mar 2017 17:07:04 +0100 Subject: [PATCH 2/5] Works for now. --- modules/user.py | 11 ++++++----- render/html/user.py | 6 +++--- render/json/user.py | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/user.py b/modules/user.py index 9e3c260..04ac8fa 100644 --- a/modules/user.py +++ b/modules/user.py @@ -301,6 +301,7 @@ def login(self, skey="", *args, **kwargs): class TimeBasedOTP(object): windowSize = 5 + method = u"X-VIUR-2FACTOR-TimeBasedOTP" def __init__(self, userModule, modulePath): super(TimeBasedOTP, self).__init__() @@ -309,7 +310,7 @@ def __init__(self, userModule, modulePath): @classmethod def get2FactorMethodName(*args,**kwargs): - return u"X-VIUR-2FACTOR-TimeBasedOTP" + return TimeBasedOTP.method def canHandle(self, userKey): user = db.Get(userKey) @@ -326,7 +327,7 @@ def startProcessing(self, userKey): "timestamp": time(), "failures": 0} session.current.markChanged() - return self.userModule.render.loginSecondFactor("X-VIUR-2FACTOR-TimeBasedOTP") + return self.userModule.render.loginSecondFactor(TimeBasedOTP.method) return None @@ -369,7 +370,7 @@ def otp(self, otptoken = None, skey = None, *args, **kwargs): raise errors.Forbidden() if otptoken is None: - self.userModule.render.edit(self.otpSkel(), **kwargs) + return self.userModule.render.loginSecondFactor(TimeBasedOTP.method) if not securitykey.validate(skey): raise errors.PreconditionFailed() @@ -382,7 +383,7 @@ def otp(self, otptoken = None, skey = None, *args, **kwargs): otptoken = int(otptoken) except: # We got a non-numeric token - this can't be correct - self.userModule.render.edit(self.otpSkel(), **kwargs) + return self.userModule.render.loginSecondFactor(TimeBasedOTP.method, secondFactorFailed=True) #logging.debug(otptoken) #logging.debug(validTokens) @@ -407,7 +408,7 @@ def otp(self, otptoken = None, skey = None, *args, **kwargs): session.current["_otp_user"] = token session.current.markChanged() - return self.userModule.render.edit(self.otpSkel(), loginFailed=True, **kwargs) + return self.userModule.render.loginSecondFactor(TimeBasedOTP.method, secondFactorFailed=True) def updateTimeDrift(self, userKey, idx): """ diff --git a/render/html/user.py b/render/html/user.py index 1666f9b..83731ad 100644 --- a/render/html/user.py +++ b/render/html/user.py @@ -21,16 +21,16 @@ def login(self, skel, tpl=None, **kwargs): return self.edit(skel, tpl=tpl, **kwargs) - def loginSecondFactor(self, factor, tpl=None, **kwargs): + def loginSecondFactor(self, method, tpl=None, **kwargs): if "loginSecondFactorTemplate" in dir(self.parent): tpl = tpl or self.parent.loginSecondFactorTemplate else: tpl = tpl or self.loginSecondFactorTemplate template= self.getEnv().get_template(self.getTemplateFileName(tpl)) - return template.render(factor=factor, **kwargs) + return template.render(method=method, **kwargs) - def loginSuccess(self, tpl=None, **kwargs): + def loginSucceeded(self, tpl=None, **kwargs): if "loginSuccessTemplate" in dir( self.parent ): tpl = tpl or self.parent.loginSuccessTemplate else: diff --git a/render/json/user.py b/render/json/user.py index 62be629..0319fdf 100644 --- a/render/json/user.py +++ b/render/json/user.py @@ -11,8 +11,8 @@ def login(self, skel, **kwargs): return self.edit(skel, **kwargs) - def loginSecondFactor(self, factor, **kwargs): - return json.dumps(factor) + def loginSecondFactor(self, method, **kwargs): + return json.dumps(method) def loginSucceeded(self, **kwargs): return json.dumps("OKAY") From d913eca46b361218ac0397e314a93519a67bcec9 Mon Sep 17 00:00:00 2001 From: Jan Max Meyer Date: Wed, 29 Mar 2017 15:17:45 +0200 Subject: [PATCH 3/5] Correct pulling through of module-defined values to the JSON renderer, with full backward compatibility. Refactoring the password bone's error messages. --- bones/passwordBone.py | 20 +++++++++++++------- render/json/default.py | 29 +++++++++++++++-------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/bones/passwordBone.py b/bones/passwordBone.py index 976b957..785cc8b 100644 --- a/bones/passwordBone.py +++ b/bones/passwordBone.py @@ -60,14 +60,19 @@ class passwordBone( stringBone ): def isInvalid(self, value): if not value: return False - if len(value) Date: Fri, 19 May 2017 14:08:39 +0200 Subject: [PATCH 4/5] Fixed for current live version --- modules/user.py | 33 ++++++++++++++++++++++----------- render/html/user.py | 16 +++------------- render/json/default.py | 32 +++++++++++++------------------- render/json/user.py | 3 --- 4 files changed, 38 insertions(+), 46 deletions(-) diff --git a/modules/user.py b/modules/user.py index 04ac8fa..9d7f1eb 100644 --- a/modules/user.py +++ b/modules/user.py @@ -301,7 +301,6 @@ def login(self, skey="", *args, **kwargs): class TimeBasedOTP(object): windowSize = 5 - method = u"X-VIUR-2FACTOR-TimeBasedOTP" def __init__(self, userModule, modulePath): super(TimeBasedOTP, self).__init__() @@ -310,12 +309,18 @@ def __init__(self, userModule, modulePath): @classmethod def get2FactorMethodName(*args,**kwargs): - return TimeBasedOTP.method + return u"X-VIUR-2FACTOR-TimeBasedOTP" def canHandle(self, userKey): user = db.Get(userKey) return all([(x in user.keys() and (x=="otptimedrift" or bool(user[x]))) for x in ["otpid", "otpkey", "otptimedrift"]]) + class otpSkel( RelSkel ): + otptoken = stringBone(descr="Token", required=True, caseSensitive=False, indexed=True) + + def render(self, **params): + return self.userModule.render.edit(self.otpSkel(), action="otp", tpl=self.userModule.loginSecondFactorTemplate, **params) + def startProcessing(self, userKey): user = db.Get(userKey) if all([(x in user.keys() and user[x]) for x in ["otpid", "otpkey"]]): @@ -327,13 +332,10 @@ def startProcessing(self, userKey): "timestamp": time(), "failures": 0} session.current.markChanged() - return self.userModule.render.loginSecondFactor(TimeBasedOTP.method) + return self.render() return None - class otpSkel( RelSkel ): - otptoken = stringBone(descr="Token", required=True, caseSensitive=False, indexed=True) - def generateOtps(self, secret, timeDrift): """ Generates all valid tokens for the given secret @@ -370,7 +372,7 @@ def otp(self, otptoken = None, skey = None, *args, **kwargs): raise errors.Forbidden() if otptoken is None: - return self.userModule.render.loginSecondFactor(TimeBasedOTP.method) + return self.render() if not securitykey.validate(skey): raise errors.PreconditionFailed() @@ -382,8 +384,8 @@ def otp(self, otptoken = None, skey = None, *args, **kwargs): try: otptoken = int(otptoken) except: - # We got a non-numeric token - this can't be correct - return self.userModule.render.loginSecondFactor(TimeBasedOTP.method, secondFactorFailed=True) + # We got a non-numeric token - this cant be correct + return self.render() #logging.debug(otptoken) #logging.debug(validTokens) @@ -408,7 +410,7 @@ def otp(self, otptoken = None, skey = None, *args, **kwargs): session.current["_otp_user"] = token session.current.markChanged() - return self.userModule.render.loginSecondFactor(TimeBasedOTP.method, secondFactorFailed=True) + return self.render(secondFactorFailed=True) def updateTimeDrift(self, userKey, idx): """ @@ -434,6 +436,7 @@ class User(List): lostPasswordTemplate = "user_lostpassword" verifyEmailAddressMail = "user_verify_address" passwordRecoveryMail = "user_password_recovery" + loginSecondFactorTemplate = "user_login_secondfactor" authenticationProviders = [UserPassword, GoogleAccount] secondFactorProviders = [TimeBasedOTP] @@ -598,13 +601,19 @@ def logout(self, skey="", *args, **kwargs): raise errors.Unauthorized() if not securitykey.validate(skey): raise errors.PreconditionFailed() + + self.onLogout(user) + oldSession = {k: v for k, v in session.current.items()} # Store all items in the current session session.current.reset() + # Copy the persistent fields over for k in conf["viur.session.persistentFieldsOnLogout"]: if k in oldSession.keys(): session.current[k] = oldSession[k] + del oldSession + return self.render.logoutSuccess() @exposed @@ -615,6 +624,9 @@ def onLogin(self): usr = self.getCurrentUser() logging.info( "User logged in: %s" % usr["name"]) + def onLogout(self, usr): + logging.info( "User logged out: %s" % usr["name"]) + @exposed def edit(self, *args, **kwargs): if len( args ) == 0 and not "key" in kwargs and session.current.get("user"): @@ -627,7 +639,6 @@ def view(self, key, *args, **kwargs): """ Allow a special key "self" to reference always the current user """ - if key == "self": user = self.getCurrentUser() if user: diff --git a/render/html/user.py b/render/html/user.py index 83731ad..9d79c09 100644 --- a/render/html/user.py +++ b/render/html/user.py @@ -4,9 +4,8 @@ import default from server.skeleton import Skeleton -class Render( default.Render ): #Render user-data to xml +class Render(default.Render): #Render user-data to HTML loginTemplate = "user_login" - loginSecondFactorTemplate = "user_login_secondfactor" loginSuccessTemplate = "user_login_success" logoutSuccessTemplate = "user_logout_success" verifySuccessTemplate = "user_verify_success" @@ -21,22 +20,13 @@ def login(self, skel, tpl=None, **kwargs): return self.edit(skel, tpl=tpl, **kwargs) - def loginSecondFactor(self, method, tpl=None, **kwargs): - if "loginSecondFactorTemplate" in dir(self.parent): - tpl = tpl or self.parent.loginSecondFactorTemplate - else: - tpl = tpl or self.loginSecondFactorTemplate - - template= self.getEnv().get_template(self.getTemplateFileName(tpl)) - return template.render(method=method, **kwargs) - def loginSucceeded(self, tpl=None, **kwargs): - if "loginSuccessTemplate" in dir( self.parent ): + if "loginSuccessTemplate" in dir(self.parent): tpl = tpl or self.parent.loginSuccessTemplate else: tpl = tpl or self.loginSuccessTemplate - template= self.getEnv().get_template( self.getTemplateFileName( tpl ) ) + template = self.getEnv().get_template(self.getTemplateFileName(tpl)) return template.render(**kwargs) def logoutSuccess(self, tpl=None, **kwargs ): diff --git a/render/json/default.py b/render/json/default.py index 6acf0b0..8948c50 100644 --- a/render/json/default.py +++ b/render/json/default.py @@ -181,7 +181,7 @@ def renderSkelValues(self, skel): return res - def renderEntry(self, skel, actionName, **res): + def renderEntry(self, skel, actionName, **params): if isinstance(skel, list): vals = [self.renderSkelValues(x) for x in skel] struct = self.renderSkelStructure(skel[0]) @@ -189,11 +189,12 @@ def renderEntry(self, skel, actionName, **res): vals = self.renderSkelValues(skel) struct = self.renderSkelStructure(skel) - res.update({ + res = { + "action": actionName, "values": vals, "structure": struct, - "action": actionName - }) + "params": params + } request.current.get().response.headers["Content-Type"] = "application/json" return json.dumps(res) @@ -207,21 +208,14 @@ def add(self, skel, action = "add", **kwargs): def edit(self, skel, action = "edit", **kwargs): return self.renderEntry(skel, action, **kwargs) - def list(self, skellist, **res): - skels = [] - - for skel in skellist: - skels.append(self.renderSkelValues(skel)) - - res["skellist"] = skels - - if skellist: - res["structure"] = self.renderSkelStructure(skellist.baseSkel) - else: - res["structure"] = None - - res["cursor"] = skellist.cursor - res["action"] = "list" + def list(self, skellist, action = "list", **params): + res = { + "action": action, + "skellist": [self.renderSkelValues(skel) for skel in skellist], + "structure": self.renderSkelStructure(skellist.baseSkel) if skellist else None, + "cursor": skellist.cursor, + "params": params + } request.current.get().response.headers["Content-Type"] = "application/json" return json.dumps(res) diff --git a/render/json/user.py b/render/json/user.py index 0319fdf..0c147df 100644 --- a/render/json/user.py +++ b/render/json/user.py @@ -11,9 +11,6 @@ def login(self, skel, **kwargs): return self.edit(skel, **kwargs) - def loginSecondFactor(self, method, **kwargs): - return json.dumps(method) - def loginSucceeded(self, **kwargs): return json.dumps("OKAY") From 5a6d003a9c5df9f9fba88552a149b9c8ff40a21c Mon Sep 17 00:00:00 2001 From: Achim Schumacher Date: Mon, 29 Jul 2019 18:06:54 +0200 Subject: [PATCH 5/5] Added support for unique and multiple bones --- bones/bone.py | 31 ++++++--- bones/stringBone.py | 2 +- modules/user.py | 8 ++- skeleton.py | 151 +++++++++++++++++++++++++++----------------- 4 files changed, 122 insertions(+), 70 deletions(-) diff --git a/bones/bone.py b/bones/bone.py index 7d617ae..7c3fb57 100644 --- a/bones/bone.py +++ b/bones/bone.py @@ -366,16 +366,27 @@ def getUniquePropertyIndexValue( self, valuesCache, name ): """ Returns an hash for our current value, used to store in the uniqueProptertyValue index. """ - if valuesCache[name] is None: - return( None ) - h = hashlib.sha256() - h.update( unicode( valuesCache[name] ).encode("UTF-8") ) - res = h.hexdigest() - if isinstance( valuesCache[name], int ) or isinstance( valuesCache[name], float ) or isinstance( valuesCache[name], long ): - return("I-%s" % res ) - elif isinstance( valuesCache[name], str ) or isinstance( valuesCache[name], unicode ): - return("S-%s" % res ) - raise NotImplementedError("Type %s can't be safely used in an uniquePropertyIndex" % type(valuesCache[name]) ) + values = valuesCache[name] + if values is None: + return None + res = [] + # Make value a list so we can loop over it in any case + if not isinstance(values, list): + values = [values] + for item in values: + h = hashlib.sha256() + h.update( unicode( item ).encode("UTF-8") ) + # res = h.hexdigest() + if isinstance( item, int ) or isinstance( item, float ) or isinstance( item, long ): + res.append("I-%s" % h.hexdigest() ) + elif isinstance( item, str ) or isinstance( item, unicode ): + res.append("S-%s" % h.hexdigest() ) + else: + raise NotImplementedError("Type %s can't be safely used in an uniquePropertyIndex" % type(item) ) + if isinstance(valuesCache[name], list): + return res + # Return back to original non-list + return res[0] def getReferencedBlobs( self, valuesCache, name ): """ diff --git a/bones/stringBone.py b/bones/stringBone.py index a2b10b0..9b20ca2 100644 --- a/bones/stringBone.py +++ b/bones/stringBone.py @@ -382,5 +382,5 @@ def getUniquePropertyIndexValue(self, valuesCache, name): """ if not valuesCache[name] and not self.required: #Dont enforce a unique property on an empty string if we are required=False return( None ) - return( super( stringBone, self).getUniquePropertyIndexValue(valuesCache, name)) + return super( stringBone, self).getUniquePropertyIndexValue(valuesCache, name) diff --git a/modules/user.py b/modules/user.py index 3dc55a0..7bdfde4 100644 --- a/modules/user.py +++ b/modules/user.py @@ -153,6 +153,8 @@ def login(self, name=None, password=None, skey="", *args, **kwargs): # Check if the username matches storedUserName = res.get("name.idx", "") + if isinstance(storedUserName, list): # Multiple emails: only use main one (first in list) + storedUserName = storedUserName[0] if len(storedUserName) != len(name.lower()): isOkay = False else: @@ -202,7 +204,10 @@ def pwrecover( self, authtoken=None, skey=None, *args, **kwargs ): skel = self.lostPasswordSkel() if len(kwargs)==0 or not skel.fromClient(kwargs) or not securitykey.validate(skey): return self.userModule.render.passwdRecover(skel, tpl=self.passwordRecoveryTemplate) - user = self.userModule.viewSkel().all().filter("name.idx =", skel["name"].lower()).get() + skel_name = skel["name"] + if isinstance(skel_name, list): # Multiple emails: only use main one (first in list) + skel_name = skel_name[0] + user = self.userModule.viewSkel().all().filter("name.idx =", skel_name.lower()).get() if not user or user["status"]<10: # Unknown user or locked account skel.errors["name"] = _("Unknown user") @@ -729,6 +734,7 @@ def createNewUserIfNotExists(): """ Create a new Admin user, if the userDB is empty """ + logging.warning("createNewUserIfNotExists") userMod = getattr(conf["viur.mainApp"], "user", None) if (userMod # We have a user module and isinstance(userMod, User) diff --git a/skeleton.py b/skeleton.py index e3d73b1..324f0de 100644 --- a/skeleton.py +++ b/skeleton.py @@ -394,17 +394,21 @@ def fromClient( self, data ): if boneInstance.unique: newVal = boneInstance.getUniquePropertyIndexValue(self.valuesCache, boneName) if newVal is not None: - try: - dbObj = db.Get(db.Key.from_path("%s_%s_uniquePropertyIndex" % (self.kindName, boneName), newVal)) - if dbObj["references"] != self["key"]: #This valus is taken (sadly, not by us) - complete = False - if isinstance(boneInstance.unique, unicode): - errorMsg = _(boneInstance.unique) - else: - errorMsg = _("This value is not available") - self.errors[boneName] = errorMsg - except db.EntityNotFoundError: - pass + if not isinstance(newVal, list): + # UniquePropertyIndexValue may be a list. Make all a list so we can treat all the same... + newVal = [newVal] + for item in newVal: + try: + dbObj = db.Get(db.Key.from_path("%s_%s_uniquePropertyIndex" % (self.kindName, boneName), item)) + if dbObj["references"] != self["key"]: #This valus is taken (sadly, not by us) + complete = False + if isinstance(boneInstance.unique, unicode): + errorMsg = _(boneInstance.unique) + else: + errorMsg = _("This value is not available") + self.errors[boneName] = errorMsg + except db.EntityNotFoundError: + pass if( len(data) == 0 or (len(data) == 1 and "key" in data) @@ -607,6 +611,9 @@ def txnUpdate(key, mergeFrom, clearUpdateTag): if bone.unique: if "%s.uniqueIndexValue" % key in dbObj: oldUniqueValues[key] = dbObj["%s.uniqueIndexValue" % key] + # oldUniqeValues[key] may be a list. Enforce list type for easy usage in this func + if oldUniqueValues[key] is not None and not isinstance(oldUniqueValues[key], list): + oldUniqueValues[key] = [oldUniqueValues[key]] # Merge the values from mergeFrom in if key in mergeFrom: @@ -641,22 +648,27 @@ def txnUpdate(key, mergeFrom, clearUpdateTag): # Check if the property is really unique newUniqueValues[key] = bone.getUniquePropertyIndexValue(self.valuesCache, key) + # newUniqueValues[key] may be a list: enforce list type for easy usage in this func if newUniqueValues[key] is not None: - try: - lockObj = db.Get(db.Key.from_path( - "%s_%s_uniquePropertyIndex" % (skel.kindName, key), - newUniqueValues[key])) - - if lockObj["references"] != ourKey: - # This value has been claimed, and that not by us - - raise ValueError( - "The unique value '%s' of bone '%s' has been recently claimed!" % - (self.valuesCache[key], key)) - - except db.EntityNotFoundError: # No lockObj found for that value, we can use that - pass - dbObj["%s.uniqueIndexValue" % key] = newUniqueValues[key] + if not isinstance(newUniqueValues[key], list): + newUniqueValues[key] = [newUniqueValues[key]] + for item in newUniqueValues[key]: + try: + lockObj = db.Get(db.Key.from_path( + "%s_%s_uniquePropertyIndex" % (skel.kindName, key), + item)) + + if lockObj["references"] != ourKey: + # This value has been claimed, and that not by us + + raise ValueError( + "One of the unique values '%s' of bone '%s' has been recently claimed!" % + (self.valuesCache[key], key)) + + except db.EntityNotFoundError: # No lockObj found for that value, we can use that + pass + dbObj["%s.uniqueIndexValue" % key] = newUniqueValues[key] \ + if bone.multiple else newUniqueValues[key][0] else: if "%s.uniqueIndexValue" % key in dbObj: @@ -715,36 +727,51 @@ def txnUpdate(key, mergeFrom, clearUpdateTag): for key, bone in skel.items(): if bone.unique: # Update/create/delete missing lock-objects - if key in oldUniqueValues and oldUniqueValues[key] != newUniqueValues[key]: - - # We had an old lock and its value changed - try: - # Try to delete the old lock - oldLockObj = db.Get(db.Key.from_path( - "%s_%s_uniquePropertyIndex" % (skel.kindName, key), - oldUniqueValues[key])) - if oldLockObj["references"] != ourKey: - # We've been supposed to have that lock - but we don't. - # Don't remove that lock as it now belongs to a different entry - logging.critical( - "Detected Database corruption! A Value-Lock had been reassigned!") - else: - # It's our lock which we don't need anymore - db.Delete(db.Key.from_path( - "%s_%s_uniquePropertyIndex" % ( - skel.kindName, key), - oldUniqueValues[key])) - except db.EntityNotFoundError as e: - logging.critical( - "Detected Database corruption! Could not delete stale lock-object!") - + # oldUniqueValues and newUniqueValues are lists; newUniqueValues[key] may be None! + # logging.error("delete0") + # logging.error(key) + # logging.error(oldUniqueValues) + # logging.error(newUniqueValues) + if key in oldUniqueValues and\ + oldUniqueValues[key] is not None and\ + ( + newUniqueValues[key] is None + or (oldUniqueValues[key]) != (newUniqueValues[key]) + ): + for item in oldUniqueValues[key]: + if newUniqueValues[key] is None or item not in newUniqueValues[key]: + # We had an old lock and its value changed + try: + # Try to delete the old lock + # Loop over old lockObjs and delete those not equal to new locks + oldLockObjKey = db.Key.from_path( + "%s_%s_uniquePropertyIndex" % (skel.kindName, key), + item) + oldLockObj = db.Get(oldLockObjKey) + # logging.error("delete1") + # logging.error(dbObj.keys()) + if oldLockObj["references"] != ourKey: + # We've been supposed to have that lock - but we don't. + # Don't remove that lock as it now belongs to a different entry + logging.critical( + "Detected Database corruption! A Value-Lock had been reassigned!") + else: + # It's our lock which we don't need anymore + # logging.error("delete2") + # logging.error(oldLockObjKey) + db.Delete(oldLockObjKey) + except db.EntityNotFoundError as e: + logging.critical( + "Detected Database corruption! Could not delete stale lock-object!") if newUniqueValues[key] is not None: - # Lock the new value - newLockObj = db.Entity( - "%s_%s_uniquePropertyIndex" % (skel.kindName, key), - name=newUniqueValues[key]) - newLockObj["references"] = str(dbObj.key()) - db.Put(newLockObj) + for item in newUniqueValues[key]: + # Lock the new value + # TODO: We could reduce write-ops if we won't write items existent in oldUniqeValues + newLockObj = db.Entity( + "%s_%s_uniquePropertyIndex" % (skel.kindName, key), + name=item) + newLockObj["references"] = str(dbObj.key()) + db.Put(newLockObj) return str(dbObj.key()), dbObj, skel @@ -844,10 +871,18 @@ def txnDelete(key, skel): # Ensure that we delete any value-lock objects remaining for this entry if bone.unique: try: + # dbObj["%s.uniqueIndexValue" % boneName] may now be a list: loop over all items + # and delete each s_uniquePropertyIndex db entry if "%s.uniqueIndexValue" % boneName in dbObj: - db.Delete(db.Key.from_path( - "%s_%s_uniquePropertyIndex" % (skel.kindName, boneName), - dbObj["%s.uniqueIndexValue" % boneName])) + oldUniqueValues = dbObj["%s.uniqueIndexValue" % boneName] + if oldUniqueValues is not None and not isinstance(oldUniqueValues, list): + oldUniqueValues = [oldUniqueValues] + logging.error("x3") + logging.error(oldUniqueValues) + for item in oldUniqueValues: + db.Delete(db.Key.from_path( + "%s_%s_uniquePropertyIndex" % (skel.kindName, boneName), + item)) except db.EntityNotFoundError: raise