# Copyright (c) Fraunhofer MEVIS, Germany. All rights reserved.
# **InsertLicense** code author="Jan-Martin Kuhnigk"
# exempt this module from Python module reloading, which frequently breaks
# exception handling, isinstance checks and enum value comparisons
_mlab_do_not_reload = True # value is currently ignored, but it must be defined
from collections import OrderedDict
from collections.abc import Mapping
import json, uuid
from functools import partial
[docs]def dump_dict( d, msg_prefix=" " ):
"""
Just for nicer debug printing
:param d: Dictionary to print
:param msg_prefix: Additional message prefix
:return: None
"""
for k, v in sorted( d.items( d ) ):
print( "{}{}: {}".format( msg_prefix, k, v ) )
[docs]def to_ParameterInfo( kv ):
"""
Recursively converts given key value store kv (possibly a dict) into a new "ParameterInfo" object.
:return: ParameterInfo with the same keys and values as the given source dictionary
"""
from parameter_info.parameter_info import ParameterInfo
return _convertMappingRecursively( kv, ParameterInfo, to_ParameterInfo )
[docs]def to_dict( kv ):
"""
Recursively converts given key value store kv (possibly a ParameterInfo object) into an "ordinary" dict.
Use especially if kv contains nodes with keys with indirect value access to replace those with the actual values.
:return: Pure dict with "scalar" (non-query) values
"""
return _convertMappingRecursively( kv, dict, to_dict )
def _convertMappingRecursively( kv, Target_type, convert_fct ):
d = Target_type()
for k, v in kv.items():
if isinstance( v, Mapping ):
d[ k ] = convert_fct( v )
else:
d[ k ] = v
return d
[docs]def to_flat_dict( kv, general_prefix='', group_prefix="", group_postfix=".", omitEmptyDictionaries = False ):
"""
Returns a flattened (i.e. depth 1) version of the given dictionary, combining keys so that the descend path is
clear. Example:
.. code-block:: python
to_flat_dict( { "a":1, "b": { "x":10, "y":{ "z": 20 }}, c:{} ) ->
{
"a":1,
"b.x": 10,
"b.y.z": 20
"c": {}
}
Note that the empty dictionary c would be omitted if omitEmptyDictionaries were set to False.
:param kv: Dictionary to flatten
:param general_prefix: Prefix for each key of the resulting dict
:param group_prefix: Prefix for each group's key in combined keys
:param group_postfix: Postfix for each group's key in combined keys
:param omitEmptyDictionaries: If set to True, an entry with an empty dictionary does not show up in the flat version
:return: Flat dict containing the composed keys and their values
"""
items = [ ]
for k, v in kv.items():
if isinstance( v, Mapping ) and ( omitEmptyDictionaries or len( v ) ):
new_key = general_prefix + group_prefix + k + group_postfix
items.extend( to_flat_dict( v, general_prefix=new_key,
group_prefix=group_prefix, group_postfix=group_postfix ).items() )
else:
new_key = general_prefix + k
items.append( (new_key, v) )
return dict( items )
[docs]def to_flat_ordered_dict( kv, general_prefix='', group_prefix="", group_postfix="." ):
"""
Like to_flat_dict, but returning an OrderedDict sorted by keys
:param kv: Dictionary to flatten
:param general_prefix: Prefix for each key of the resulting dict
:param group_prefix: Prefix for each group's key in combined keys
:param group_postfix: Postfix for each group's key in combined keys
:return: OrderedDict containing the composed keys in alphabetical order (and their values)
"""
return to_ordered_dict(
to_flat_dict( kv, general_prefix=general_prefix, group_prefix=group_prefix, group_postfix=group_postfix ) )
[docs]def to_ordered_dict( kv ):
"""
Convenience method to convert a dictionary to an OrderedDict sorted by key, but without
recursing into dictionary children.
:param kv: Source dictionary
:return: Ordered dictionary
"""
return OrderedDict( sorted( kv.items(), key=lambda x: x[ 0 ] ) )
[docs]def get_ordered_copy( kv ):
"""
Convenience method to convert any kind of dictionary (could also be a ParameterInfo) into an ordered copy of
itself sorted by key, with recursing into the dictionary's children
:param kv: Source dictionary
:return: Copied dictionary
"""
return _convertMappingRecursively( to_ordered_dict( kv ), type( kv ), partial( get_ordered_copy ))
class TypeCleaner( object ):
# List here all elementary types that are directly supported by JSON.dumps
__SUPPORTED_ELEMENTARY_TYPES = (
int,
float,
str
)
# List here conversion functions for each not natively supported type. Mapping is done
# by string, as we did not want to add (e.g. numpy) dependencies here. In the future, this
# may be refactored
__SUPPORTED_TYPE_CONVERSIONS = { "int64": int, "float64": float, "datetime": str }
# In order not to break tests that use sentinels
__IGNORED_TYPE_NAMES = { "_SentinelObject" }
@classmethod
def get_json_compatible_copy( cls, value, key="", raiseOnError=False ):
"""
Creates a copy of the provided data structure, which may be any composition of elementary and container types.
Will convert known types that are not supported by Json.dumps() into something that is.
:param value: Value to copy (elementary or container)
:param key: Optional key identifier used only for error logging (simplifies finding unexpected datatypes in a container)
:param raiseOnError: If enabled, a TypeError is raised for each conversion error. Otherwise, only an info message is printed,\
and the unconvertible value is put out as a string with a _unsupported_type_<typename> postfix.
:return: (hopefully) json compatible copy of the provided data structure
"""
if cls.__has_supported_conversion_type( value ):
result = cls.__get_json_compatible_copy_of_supported_type( value )
elif cls.__has_ignored_type( value ):
result = value
elif isinstance( value, dict ):
result = cls.__get_json_compatible_copy_of_dict( value, key )
elif isinstance( value, (tuple, list, set) ):
result = cls.__get_json_compatible_copy_of_list_types( value, key )
elif isinstance( value, cls.__SUPPORTED_ELEMENTARY_TYPES ):
result = value # nothing to do
elif value is None:
result = None
elif cls.__get_typename(value) == 'ndarray':
result = cls.__get_json_compatible_copy_of_list_types( value.tolist(), key )
else:
errMsg = "get_json_compatible_copy error: Unsupported type {} found for key {}!".format(
cls.__get_typename( value ), key )
if raiseOnError:
raise TypeError( errMsg )
else:
print( errMsg )
result = "{}_unsupported_type_{}".format( value, cls.__get_typename( value ) )
return result
@classmethod
def __has_ignored_type( cls, value ):
return cls.__get_typename( value ) in cls.__IGNORED_TYPE_NAMES
@classmethod
def __get_json_compatible_copy_of_supported_type( cls, value ):
return cls.__SUPPORTED_TYPE_CONVERSIONS[ cls.__get_typename( value ) ]( value )
@classmethod
def __has_supported_conversion_type( cls, value ):
return cls.__get_typename( value ) in cls.__SUPPORTED_TYPE_CONVERSIONS
@classmethod
def __get_typename( cls, value ):
return type( value ).__name__
@classmethod
def __get_json_compatible_copy_of_dict( cls, value, key ):
result = { }
for k, v in value.items():
result[ k ] = cls.get_json_compatible_copy( v, key = "{}.{}".format( key, k ) )
return result
@classmethod
def __get_json_compatible_copy_of_list_types( cls, value, key ):
result = [ cls.get_json_compatible_copy( v, key = "{}[{}]".format( key, i ) ) for i, v in enumerate( value ) ]
if isinstance( value, tuple ):
result = tuple( result )
elif isinstance( value, set ):
result = set( result )
return result
# --------------------------------------------------------------------
[docs]class SelectiveNoIndentEncoder( json.JSONEncoder ):
"""
Replacement for the default JSONEncoder that will prevent indentation for values wrapped in a
SelectiveNoIndentEncoder.NoIndent object. Note that using a NoIndent on a composed object, none of the
children in its subtree will be indented (additional NoIndent objects there will be tolerated).
Adapted from https://stackoverflow.com/a/25935321/5226368
NOTE: Only works correctly with json.dumps, not json.dump! In the latter case, it will have no effect.
"""
[docs] class NoIndent( object ):
"""
Use this class to wrap values that you want to exclude from indentation.
"""
def __init__( self, value ):
self.value = value
def __init__( self, *args, **kwargs ):
super( SelectiveNoIndentEncoder, self ).__init__( *args, **kwargs )
self.kwargs = dict( kwargs )
del self.kwargs[ 'indent' ]
self._replacement_map = { }
[docs] def default( self, o ):
if isinstance( o, SelectiveNoIndentEncoder.NoIndent ):
key = uuid.uuid4().hex
self._replacement_map[ key ] = json.dumps( o.value, cls=SelectiveNoIndentEncoder, **self.kwargs )
return "@@%s@@" % (key,)
else:
return super( SelectiveNoIndentEncoder, self ).default( o )
[docs] def encode( self, o ):
result = super( SelectiveNoIndentEncoder, self ).encode( o )
for k, v in self._replacement_map.items():
result = result.replace( '"@@%s@@"' % (k,), v )
return result
[docs]class IndentOnlyNestedSequencesJsonEncoder( SelectiveNoIndentEncoder ):
"""
Replacement for the default JSONEncoder that will write non-nested lists, tuples or sets in a single line even
with indent > 0.
NOTE: Only works correctly with json.dumps, not json.dump! In the latter case, it will have no effect.
"""
[docs] class MarkElementarySequences( TypeCleaner ):
"""
Modifies the TypeCleaner to mark all non-nested sequences by encapsulating them in the NoIndent wrapper.
"""
[docs] @classmethod
def get_json_compatible_copy( cls, value, key="", raiseOnError=False ):
if cls._is_non_nested_sequence( value ):
return SelectiveNoIndentEncoder.NoIndent( value )
else:
return super().get_json_compatible_copy( value, key=key, raiseOnError=raiseOnError )
@classmethod
def _is_non_nested_sequence( cls, value ):
if not isinstance( value, (tuple, list, set) ):
return False
for v in value:
if not isinstance( v, (str, int, float) ):
return False
return True
[docs] def encode( self, o ):
return super().encode( self.MarkElementarySequences.get_json_compatible_copy( o ) )