diff --git a/src/verifai/features/features.py b/src/verifai/features/features.py index 8634ceb..e57ce7c 100644 --- a/src/verifai/features/features.py +++ b/src/verifai/features/features.py @@ -9,7 +9,9 @@ import random import itertools import functools +from abc import ABC, abstractmethod from collections import OrderedDict, namedtuple +from collections.abc import Sequence import numpy as np @@ -964,19 +966,26 @@ def distance(self, valueA, valueB): else: return self.distanceMetric(valueA, valueB) - @cached_property - def fixedDomains(self): + @staticmethod + def _timeExpandDomain(domain, timeBound): + return domain + + def fixedDomains(self, timeBound): """Return the fixed-length Domains associated with this feature.""" + timeExpandedDomain = self._timeExpandDomain(self.domain, timeBound) if timeBound is not None else self.domain + if not self.lengthDomain: - return self.domain - domains = {} - for length in self.lengthDomain: - length = length[0] - domains[length] = Array(self.domain, (length,)) + domains = timeExpandedDomain + else: + domains = {} + for length in self.lengthDomain: + length = length[0] + domains[length] = Array(timeExpandedDomain, (length,)) + return domains def __repr__(self): - rep = f'Feature({self.domain}' + rep = f'{self.__class__.__name__}({self.domain}' if self.distribution is not None: rep += f', distribution={self.distribution}' if self.lengthDomain is not None: @@ -987,11 +996,158 @@ def __repr__(self): rep += f', distanceMetric={self.distanceMetric}' return rep + ')' -### Feature spaces +class TimeSeriesFeature(Feature): + """A feature with a value at each timestep of a simulation.""" + @staticmethod + def _timeExpandDomain(domain, timeBound): + return Array(domain, (timeBound,)) + +### Feature spaces and Samples +class _SampleBase(ABC): + def __init__(self, space, dynamicSampleLengths): + self.space = space + self.dynamicSampleLengths = dynamicSampleLengths + + self._dynamicSamples = [] + + @property + @abstractmethod + def staticSample(self): + pass + + def __getattr__(self, attr): + space = super().__getattribute__("space") + if attr in space.staticFeatureNamed: + return getattr(self.staticSample, attr) + elif attr in space.dynamicFeatureNamed: + class DynamicFeatureHelper(Sequence): + def __init__(self, dynamicSampleHistory, attr): + self._dynamicSamples = dynamicSampleHistory + self.attr = attr + + def __getitem__(self, i): + if i > len(self._dynamicSamples)-1: + raise IndexError("Attempting to access dynamic sample value that has not been sampled.") + return getattr(self._dynamicSamples[i], self.attr) + + def __len__(self): + return len(self._dynamicSamples) + + return DynamicFeatureHelper(self._dynamicSamples, attr) + else: + return super().__getattribute__(attr) + + +class Sample(_SampleBase): + """A sample from a feature space, containing a static point and able to generate dynamic points. + + Args: + space (FeatureSpace): The feature space this Sample was sampled from. + dynamicSampleLengths (dict): A dictionary containing the lengths of each dynamic feature + with a length domain. + """ + def __init__(self, space, dynamicSampleLengths): + super().__init__(space, dynamicSampleLengths) + + @property + @abstractmethod + def staticSample(self): + pass + + @abstractmethod + def _getDynamicSample(self, info): + pass + + def getDynamicSample(self, info=None): + sample = self._getDynamicSample(info) + self._dynamicSamples.append(sample) + return sample + + def complete(self, rho): + """Super method should be called after providing rho to the sampler to return a `CompletedSample`""" + return CompletedSample(self.staticSample, self._dynamicSamples, self.space, self.dynamicSampleLengths) + + +class CompletedSample(_SampleBase): + """A completed sample from a feature space, containing the static point the history of sampled dynamic points. + + This class is returned when calling `complete` on a `Sample`, and is useful for storage, comparisons, hashing, etc... + """ + + def __init__(self, staticSample, dynamicSamples, space, dynamicSampleLengths): + super().__init__(space, dynamicSampleLengths) + self._staticSample = staticSample + self._dynamicSamples = tuple(dynamicSamples) + + @property + def staticSample(self): + return self._staticSample + + @property + def dynamicSamples(self): + return self._dynamicSamples + + def __eq__(self, other): + if not isinstance(other, CompletedSample): + return False + + return self.staticSample == other.staticSample and self.dynamicSamples == other.dynamicSamples + + def __hash__(self): + return hash((self.staticSample, self.dynamicSamples)) + +class _PrecomputedSample(Sample): + """A precompleted sample, which has fully computed static and dynamic points. + + Note: This class is an implementation detail, and one should only assume the `Sample` API + with it. + + Args: + space (FeatureSpace): The feature space this Sample was sampled from. + staticSample: The static point for this sample + dynamicSampleList: A list of the dynamic points for this sample. + completeCallback: A callback that is called with the value passed to complete. + dynamicSampleLengths (dict): A dictionary containing the lengths of each dynamic feature + with a length domain. + """ + def __init__(self, space, staticSample, dynamicSampleList, completeCallback, dynamicSampleLengths): + super().__init__(space, dynamicSampleLengths) + self._staticSample = staticSample + self._dynamicSampleList = dynamicSampleList + self._completeCallback = completeCallback + self._i = 0 + + @property + def staticSample(self): + return self._staticSample + + def _getDynamicSample(self, info): + if self.space.timeBound == 0: + raise RuntimeError("Called `getDynamicSample` with `timeBound` of `FeatureSpace` set to 0") + + if self._i >= self.space.timeBound: + raise RuntimeError("Exceeded `timeBound` of `FeatureSpace`") + + assert self._i < len(self._dynamicSampleList) + + dynamic_sample = self._dynamicSampleList[self._i] + self._i += 1 + + return dynamic_sample + + def complete(self, rho): + self._completeCallback(rho) + return super().complete(rho) class FeatureSpace: """A space consisting of named features. + Args: + features (Iterable): An iterable of the `Feature` objects in this space. + distanceMetric (function; optional): An optional distance metric to be used with this space. + timeBound (int; optional): An upper bound on the number of timesteps of a simulation using + this space. + .. testcode:: FeatureSpace({ @@ -1001,22 +1157,37 @@ class FeatureSpace: }) """ - def __init__(self, features, distanceMetric=None): + def __init__(self, features, distanceMetric=None, timeBound=0): self.namedFeatures = tuple(sorted(features.items(), key=lambda i: i[0])) self.featureNamed = OrderedDict(self.namedFeatures) self.features = tuple(self.featureNamed.values()) - self.makePoint = namedtuple('SpacePoint', self.featureNamed.keys()) + + self.staticFeatureNamed = OrderedDict({name: feat for name, feat in self.featureNamed.items() + if not isinstance(feat, TimeSeriesFeature)}) + self.dynamicFeatureNamed = OrderedDict({name: feat for name, feat in self.featureNamed.items() + if isinstance(feat, TimeSeriesFeature)}) + + self.hasTimeSeries = len(self.dynamicFeatureNamed) > 0 + + self.makeStaticPoint = namedtuple('StaticSpacePoint', self.staticFeatureNamed) + self.makeDynamicPoint = namedtuple('DynamicSpacePoint', self.dynamicFeatureNamed) + self.distanceMetric = distanceMetric + self.timeBound = timeBound + + if len(self.dynamicFeatureNamed) > 0 and self.timeBound == 0: + raise ValueError("must specify timeBound when creating a FeatureSpace with a TimeSeriesFeature") @cached_property def domains(self): - """Return the domain or domains associated with this space. + """Return the expanded domain or domains associated with this space. Returns a pair consisting of the Domain of all lengths of feature lists, plus a dict mapping each (flattened) point in that Domain to the corresponding Domain of other features. If the FeatureSpace has no feature lists, then returns (None, dom) where dom is the fixed Domain - of all features. + of all features. If any Features are TimeSeriesFeatures then they are + expanded to a max of timeBound. """ fixedDomains = {} lengthDomains = {} @@ -1024,9 +1195,9 @@ def domains(self): for name, feature in self.namedFeatures: if feature.lengthDomain: lengthDomains[name] = feature.lengthDomain - variableDomains[name] = feature.fixedDomains + variableDomains[name] = feature.fixedDomains(self.timeBound) else: - fixedDomains[name] = feature.domain + fixedDomains[name] = feature._timeExpandDomain(feature.domain, self.timeBound) if len(lengthDomains) == 0: return (None, Struct(fixedDomains)) lengthDomain = Struct(lengthDomains) @@ -1056,16 +1227,18 @@ def flatten(self, point, fixedDimension=False): """Flatten a point in this space. See Domain.flatten. If fixedDimension is True, the point is flattened out as if all feature - lists had their maximum lengths, with None as a placeholder. This means - that all points in the space will flatten to the same length. + lists had their maximum lengths and time steps, with None as a placeholder. + This means that all points in the space will flatten to the same length. """ + assert isinstance(point, CompletedSample) + flattened = [] - for feature, value in zip(self.features, point): + for feature, value in zip(self.staticFeatureNamed.values(), point.staticSample): domain = feature.domain if feature.lengthDomain: length = len(value) flattened.append(length) - fixedDomain = feature.fixedDomains[length] + fixedDomain = feature.fixedDomains(None)[length] fixedDomain.flattenOnto(value, flattened) if fixedDimension: # add padding to maximum length sizePerElt = domain.flattenedDimension @@ -1074,6 +1247,43 @@ def flatten(self, point, fixedDimension=False): flattened.append(None) else: domain.flattenOnto(value, flattened) + + if self.hasTimeSeries: + duration = len(point.dynamicSamples) + flattened.append(duration) + + for feature_i, f in enumerate(self.dynamicFeatureNamed.items()): + feature_name, feature = f + domain = feature.domain + sizePerElt = domain.flattenedDimension + + if feature.lengthDomain: + length = point.dynamicSampleLengths[feature_name] + else: + length = None + + flattened.append(length) + + for dynamic_point in point.dynamicSamples: + value = dynamic_point[feature_i] + + if length is None: + domain.flattenOnto(value, flattened) + else: + fixedDomain = feature.fixedDomains(None)[length] + fixedDomain.flattenOnto(value, flattened) + if fixedDimension: + needed = (feature.maxLength - length) * sizePerElt + for i in range(needed): + flattened.append(None) + + if fixedDimension: + needed = (self.timeBound - len(point.dynamicSamples)) * feature.maxLength * sizePerElt + flattened += [None for _ in range(needed)] + + flattened_point = tuple(flattened) + if fixedDimension: + assert len(flattened_point) == self.fixedFlattenedDimension return tuple(flattened) @cached_property @@ -1083,13 +1293,16 @@ def fixedFlattenedDimension(self): Also an upper bound on the length of the vector returned by flatten by default, when fixedDimension=False.""" dim = 0 + if self.hasTimeSeries: + dim += 1 # Timesteps for feature in self.features: domain = feature.domain + timeMult = self.timeBound if isinstance(feature, TimeSeriesFeature) else 1 if feature.lengthDomain: dim += 1 # dimension storing length of the feature list - dim += feature.maxLength * domain.flattenedDimension + dim += timeMult * feature.maxLength * domain.flattenedDimension else: - dim += domain.flattenedDimension + dim += timeMult * domain.flattenedDimension return dim def meaningOfFlatCoordinate(self, index, pointName='point'): @@ -1100,7 +1313,7 @@ def meaningOfFlatCoordinate(self, index, pointName='point'): have different meaning depending on the lengths of feature lists. """ assert 0 <= index < self.fixedFlattenedDimension - for name, feature in self.namedFeatures: + for name, feature in self.staticFeatureNamed.items(): domain = feature.domain if feature.lengthDomain: if index == 0: @@ -1120,6 +1333,35 @@ def meaningOfFlatCoordinate(self, index, pointName='point'): return domain.meaningOfFlatCoordinate(index, pointName=subPoint) index -= domain.flattenedDimension + + if index == 0: + return f'len({pointName}.dynamicSampleHistory)' + index -= 1 + + for name, feature in self.dynamicFeatureNamed.items(): + domain = feature.domain + if feature.lengthDomain: + if index == 0: + return f'{pointName}.dynamicSampleLengths["{name}"]' + else: + index -= 1 + for time_i in range(self.timeBound): + elem = index // domain.flattenedDimension + if elem < feature.maxLength: + subIndex = index % domain.flattenedDimension + subPoint = f'{pointName}.{name}[{time_i}][{elem}]' + return domain.meaningOfFlatCoordinate(subIndex, + pointName=subPoint) + index -= feature.maxLength * domain.flattenedDimension + else: + index -= 1 + for time_i in range(self.timeBound): + if index < domain.flattenedDimension: + subPoint = f'{pointName}.{name}[{time_i}]' + return domain.meaningOfFlatCoordinate(index, + pointName=subPoint) + index -= domain.flattenedDimension + raise RuntimeError('impossible index arithmetic') def pandasIndexForFlatCoordinate(self, index): @@ -1128,7 +1370,7 @@ def pandasIndexForFlatCoordinate(self, index): See meaningOfFlatCoordinate, and Domain.pandasIndexForFlatCoordinate. """ assert 0 <= index < self.fixedFlattenedDimension - for name, feature in self.namedFeatures: + for name, feature in self.staticFeatureNamed.items(): domain = feature.domain if feature.lengthDomain: if index == 0: @@ -1146,15 +1388,42 @@ def pandasIndexForFlatCoordinate(self, index): panda = domain.pandasIndexForFlatCoordinate(index) return (name,) + panda index -= domain.flattenedDimension + + if index == 0: + return ("dynamicSamples", "length") + index -= 1 + + for name, feature in self.dynamicFeatureNamed.items(): + domain = feature.domain + if feature.lengthDomain: + if index == 0: + return (name, 'length') + else: + index -= 1 + for time_i in range(self.timeBound): + elem = index // domain.flattenedDimension + if elem < feature.maxLength: + subIndex = index % domain.flattenedDimension + panda = domain.pandasIndexForFlatCoordinate(subIndex) + return (name, elem) + panda + index -= feature.maxLength * domain.flattenedDimension + else: + for time_i in range(self.timeBound): + if index < domain.flattenedDimension: + panda = domain.pandasIndexForFlatCoordinate(index) + return (name,) + panda + index -= domain.flattenedDimension + raise RuntimeError('impossible index arithmetic') + def coordinateIsNumerical(self, index): """Whether the value of a coordinate is intrinsically numerical. See meaningOfFlatCoordinate, and Domain.coordinateIsNumerical. """ assert 0 <= index < self.fixedFlattenedDimension - for name, feature in self.namedFeatures: + for name, feature in self.staticFeatureNamed.items(): domain = feature.domain if feature.lengthDomain: if index == 0: @@ -1170,29 +1439,94 @@ def coordinateIsNumerical(self, index): if index < domain.flattenedDimension: return domain.coordinateIsNumerical(index) index -= domain.flattenedDimension + + if index == 0: + return True + index -= 1 + + for name, feature in self.dynamicFeatureNamed.items(): + domain = feature.domain + if feature.lengthDomain: + if index == 0: + return True + else: + index -= 1 + for time_i in range(self.timeBound): + elem = index // domain.flattenedDimension + if elem < feature.maxLength: + subIndex = index % domain.flattenedDimension + return domain.coordinateIsNumerical(subIndex) + index -= feature.maxLength * domain.flattenedDimension + else: + for time_i in range(self.timeBound): + if index < domain.flattenedDimension: + return domain.coordinateIsNumerical(index) + index -= domain.flattenedDimension + raise RuntimeError('impossible index arithmetic') def unflatten(self, coords, fixedDimension=False): """Unflatten a tuple of coordinates to a point in this space.""" - values = [] + staticValues = [] iterator = iter(coords) - for feature in self.features: + + for feature in self.staticFeatureNamed.values(): domain = feature.domain if feature.lengthDomain: length = next(iterator) - fixedDomain = feature.fixedDomains[length] - values.append(fixedDomain.unflattenIterator(iterator)) + fixedDomain = feature.fixedDomains(None)[length] + staticValues.append(fixedDomain.unflattenIterator(iterator)) if fixedDimension: # consume padding sizePerElt = domain.flattenedDimension needed = (feature.maxLength - length) * sizePerElt for i in range(needed): next(iterator) else: - values.append(domain.unflattenIterator(iterator)) - return self.makePoint(*values) + staticValues.append(domain.unflattenIterator(iterator)) + + staticSample = self.makeStaticPoint(*staticValues) + + if self.hasTimeSeries: + duration = next(iterator) + + dynamicValuesList = [[] for _ in range(duration)] + dynamicSampleLengths = {} + + for feature_name, feature in self.dynamicFeatureNamed.items(): + domain = feature.domain + sizePerElt = domain.flattenedDimension + length = next(iterator) + dynamicSampleLengths[feature_name] = length + + for time_i in range(duration): + if length is None: + dynamicValuesList[time_i].append(domain.unflattenIterator(iterator)) + else: + fixedDomain = feature.fixedDomains(None)[length] + dynamicValuesList[time_i].append(fixedDomain.unflattenIterator(iterator)) + if fixedDimension: # consume padding + needed = (feature.maxLength - length) * sizePerElt + for _ in range(needed): + next(iterator) + + if fixedDimension: + needed = (self.timeBound - duration) * length * sizePerElt + for _ in range(needed): + next(iterator) + + dynamicSampleList = [self.makeDynamicPoint(*dynamicValues) for dynamicValues in dynamicValuesList] + else: + duration = 0 + dynamicSampleList = [] + dynamicSampleLengths = {} + + sample = CompletedSample(staticSample, dynamicSampleList, self, dynamicSampleLengths) + + return sample def __repr__(self): rep = f'FeatureSpace({self.featureNamed}' if self.distanceMetric is not None: rep += f', distanceMetric={self.distanceMetric}' return rep + ')' + diff --git a/src/verifai/samplers/domain_sampler.py b/src/verifai/samplers/domain_sampler.py index 6ead563..a1ba7b2 100644 --- a/src/verifai/samplers/domain_sampler.py +++ b/src/verifai/samplers/domain_sampler.py @@ -47,22 +47,12 @@ def update(self, sample, info, rho): """ pass - def nextSample(self, feedback=None): - """Generate the next sample, given feedback from the last sample. - - This exists only for backwards-compatibility. It has been replaced by - the getSample and update APIs. - """ - if self.last_sample is not None: - self.update(self.last_sample, self.last_info, feedback) - self.last_sample, self.last_info = self.getSample() - return self.last_sample - def __iter__(self): try: - feedback = None while True: - feedback = yield self.nextSample(feedback) + sample, info = self.getSample() + rho = yield sample + self.update(sample, info, rho) except TerminationException: return diff --git a/src/verifai/samplers/feature_sampler.py b/src/verifai/samplers/feature_sampler.py index 5890505..1719db4 100644 --- a/src/verifai/samplers/feature_sampler.py +++ b/src/verifai/samplers/feature_sampler.py @@ -8,8 +8,11 @@ import dill from dotmap import DotMap import numpy as np +from abc import ABC, abstractmethod +from contextlib import contextmanager -from verifai.features import FilteredDomain +from verifai.features import FilteredDomain, TimeSeriesFeature, Sample +from verifai.features.features import _PrecomputedSample from verifai.samplers.domain_sampler import SplitSampler, TerminationException from verifai.samplers.rejection import RejectionSampler from verifai.samplers.halton import HaltonSampler @@ -23,7 +26,7 @@ ### Samplers defined over FeatureSpaces -class FeatureSampler: +class FeatureSampler(ABC): """Abstract class for samplers over FeatureSpaces.""" def __init__(self, space): @@ -149,29 +152,14 @@ def makeDomainSampler(domain): makeRandomSampler) return LateFeatureSampler(space, RandomSampler, makeDomainSampler) - def getSample(self): - """Generate a sample, along with any sampler-specific info. - - Must return a pair consisting of the sample and arbitrary - sampler-specific info, which will be passed to the `update` - method after the sample is evaluated. - """ - raise NotImplementedError('tried to use abstract FeatureSampler') - - def update(self, sample, info, rho): + def update(self, sample_id, rho): """Update the state of the sampler after evaluating a sample.""" pass - def nextSample(self, feedback=None): - """Generate the next sample, given feedback from the last sample. - - This function exists only for backwards compatibility. It has been - superceded by the `getSample` and `update` APIs. - """ - if self.last_sample is not None: - self.update(self.last_sample, self.last_info, feedback) - self.last_sample, self.last_info = self.getSample() - return self.last_sample + @abstractmethod + def getSample(self): + """Generate a `Sample`.""" + pass def set_graph(self, graph): self.scenario.set_graph(graph) @@ -194,15 +182,22 @@ def restoreFromFile(path): def __iter__(self): try: - feedback = None while True: - feedback = yield self.nextSample(feedback) + sample = self.getSample() + rho = yield sample + sample.complete(rho) except TerminationException: return + class LateFeatureSampler(FeatureSampler): - """FeatureSampler that works by first sampling only lengths of feature - lists, then sampling from the resulting fixed-dimensional Domain. + """ FeatureSampler that greedily samples the dynamic portion of a Sample. + + LateFeatureSampler works as follows: + 1. Sample lengths of feature lists. + 2. Expand TimeSeriesFeatures into flattened features of length + space.timeBound. + 3. Sample from the resulting fixed-dimensional Domains. e.g. LateFeatureSampler(space, RandomSampler, HaltonSampler) creates a FeatureSampler which picks lengths uniformly at random and applies @@ -211,10 +206,11 @@ class LateFeatureSampler(FeatureSampler): def __init__(self, space, makeLengthSampler, makeDomainSampler): super().__init__(space) + lengthDomain, fixedDomains = space.domains if lengthDomain is None: # space has no feature lists self.lengthSampler = None - self.domainSampler = makeDomainSampler(fixedDomains) + self.domainSamplers = {None: makeDomainSampler(fixedDomains)} else: self.lengthDomain = lengthDomain self.lengthSampler = makeLengthSampler(lengthDomain) @@ -222,30 +218,62 @@ def __init__(self, space, makeLengthSampler, makeDomainSampler): point: makeDomainSampler(domain) for point, domain in fixedDomains.items() } - self.lastLength = None + + self._id_metadata_dict = {} + self._last_id = 0 + + def _get_info_id(self, info, length, sample): + self._last_id += 1 + self._id_metadata_dict[self._last_id] = (info, length, sample) + return self._last_id def getSample(self): if self.lengthSampler is None: - domainPoint, info = self.domainSampler.getSample() + length, info1 = None, None else: length, info1 = self.lengthSampler.getSample() - self.lastLength = length - domainPoint, info2 = self.domainSamplers[length].getSample() - info = (info1, info2) - return self.space.makePoint(*domainPoint), info - - def update(self, sample, info, rho): + + domainPoint, info2 = self.domainSamplers[length].getSample() + info = (info1, info2) + + sample_id = self._get_info_id(info, length, domainPoint) + complete_callback = lambda rho: self.update(sample_id, rho) + + # Make static points and iterable over dynamic points + static_features = [(k, domainPoint._asdict()[k]) for k in self.space.staticFeatureNamed] + dynamic_features = [(k, domainPoint._asdict()[k]) for k in self.space.dynamicFeatureNamed] + static_point = self.space.makeStaticPoint(*[v[1] for v in static_features]) + + dynamic_points = [] + if self.space.hasTimeSeries: + for t in range(self.space.timeBound): + raw_point_list = [] + + for f, val in dynamic_features: + if not self.space.featureNamed[f].lengthDomain: + raw_point_list.append(val[t]) + else: + raw_point_list.append(tuple(v[t] for v in val)) + + dynamic_points.append(self.space.makeDynamicPoint(*raw_point_list)) + + + dynamicSampleLengths = ({feature_name: getattr(length, feature_name)[0] + for feature_name, feature in self.space.dynamicFeatureNamed.items() + if feature.lengthDomain} + if self.lengthSampler else {}) + + return _PrecomputedSample(self.space, static_point, dynamic_points, complete_callback, dynamicSampleLengths) + + def update(self, sample_id, rho): + info, lengthPoint, domainPoint = self._id_metadata_dict[sample_id] + if self.lengthSampler is None: - self.domainSampler.update(sample, info, rho) + self.domainSamplers[None].update(domainPoint, info[1], rho) else: - self.lengthSampler.update(sample, info[0], rho) - lengths = [] - for name, feature in self.space.namedFeatures: - if feature.lengthDomain: - lengths.append((len(getattr(sample, name)),)) - lengthPoint = self.lengthDomain.makePoint(*lengths) - self.domainSamplers[lengthPoint].update(sample, info[1], rho) + self.lengthSampler.update(domainPoint, info[0], rho) + self.domainSamplers[lengthPoint].update(domainPoint, info[1], rho) ### Utilities def makeRandomSampler(domain): diff --git a/src/verifai/samplers/scenic_sampler.py b/src/verifai/samplers/scenic_sampler.py index bcea7fa..f98a0fd 100644 --- a/src/verifai/samplers/scenic_sampler.py +++ b/src/verifai/samplers/scenic_sampler.py @@ -16,7 +16,7 @@ from verifai.features import (Constant, Categorical, Real, Box, Array, Struct, Feature, FeatureSpace) -from verifai.samplers.feature_sampler import FeatureSampler +from verifai.samplers.feature_sampler import FeatureSampler, Sample from verifai.utils.frozendict import frozendict scenicMajorVersion = int(importlib.metadata.version('scenic').split('.')[0]) @@ -223,6 +223,23 @@ def spaceForScenario(scenario, ignoredProperties): }) return space, quotedParams +class ScenicSample(Sample): + def __init__(self, space, staticSample, completeCallback, dynamicSampleLengths): + super().__init__(space, dynamicSampleLengths) + self._staticSample = staticSample + self._completeCallback = completeCallback + + @property + def staticSample(self): + return self._staticSample + + def _getDynamicSample(self, info): + raise RuntimeError("ScenicSampler does not support dynamic sampling.") + + def complete(self, rho): + self._completeCallback(rho) + return super().complete(rho) + class ScenicSampler(FeatureSampler): """Samples from the induced distribution of a Scenic scenario. @@ -236,14 +253,16 @@ class ScenicSampler(FeatureSampler): def __init__(self, scenario, maxIterations=None, ignoredProperties=None): self.scenario = scenario self.maxIterations = 2000 if maxIterations is None else maxIterations + self._nextScene = None self.lastScene = None + self.lastFeedback = None if ignoredProperties is None: ignoredProperties = defaultIgnoredProperties space, self.quotedParams = spaceForScenario(scenario, ignoredProperties) super().__init__(space) @classmethod - def fromScenario(cls, path, maxIterations=None, + def fromScenario(cls, path, maxIterations=None, maxSteps=None, ignoredProperties=None, **kwargs): """Create a sampler corresponding to a Scenic program. @@ -262,25 +281,40 @@ def fromScenario(cls, path, maxIterations=None, e.g. ``params`` to override global parameters or ``model`` to set the :term:`world model`. """ + params = kwargs.setdefault("params", {}) + params["timeBound"] = maxSteps if maxSteps else 0 + scenario = scenic.scenarioFromFile(path, **kwargs) return cls(scenario, maxIterations=maxIterations, ignoredProperties=ignoredProperties) @classmethod - def fromScenicCode(cls, code, maxIterations=None, + def fromScenicCode(cls, code, maxIterations=None, maxSteps=None, ignoredProperties=None, **kwargs): """As above, but given a Scenic program as a string.""" + params = kwargs.setdefault("params", {}) + params["timeBound"] = maxSteps if maxSteps else 0 + scenario = scenic.scenarioFromString(code, **kwargs) return cls(scenario, maxIterations=maxIterations, ignoredProperties=ignoredProperties) - def nextSample(self, feedback=None): + def getSample(self): ret = self.scenario.generate( - maxIterations=self.maxIterations, feedback=feedback, verbosity=0 + maxIterations=self.maxIterations, feedback=self.lastFeedback, verbosity=0 ) + + self.lastFeedback = None self.lastScene, _ = ret + return self.pointForScene(self.lastScene) + def update(self, sample_id, rho): + assert sample_id == 0 + if self.lastFeedback is not None: + raise RuntimeError("Called `update` twice in a row (ScenicSampler does not support non-sequential sampling)") + self.lastFeedback = rho + def pointForScene(self, scene): """Convert a sampled Scenic :obj:`~scenic.core.scenarios.Scene` to a point in our feature space. @@ -314,7 +348,12 @@ def pointForScene(self, scene): params[param] = pointForValue(subdom, scene.params[originalName]) paramPoint = paramDomain.makePoint(**params) - return self.space.makePoint(objects=objPoint, params=paramPoint) + staticSample = self.space.makeStaticPoint(objects=objPoint, params=paramPoint) + + completeCallback = lambda rho: self.update(0, rho) + dynamicSampleLengths = [] + + return ScenicSample(self.space, staticSample, completeCallback, dynamicSampleLengths) @staticmethod def nameForObject(i): diff --git a/src/verifai/server.py b/src/verifai/server.py index ef4b043..b4da837 100644 --- a/src/verifai/server.py +++ b/src/verifai/server.py @@ -144,6 +144,8 @@ def __init__(self, sampling_data, monitor, options={}): sampler_params=params ) + if self.sample_space.hasTimeSeries: + raise ValueError("Sample space for `Server` cannot contain `TimeSeriesFeature`") def listen(self): client_socket, addr = self.socket.accept() @@ -176,8 +178,8 @@ def terminate(self): def close_connection(self): self.client_socket.close() - def get_sample(self, feedback): - return self.sampler.nextSample(feedback) + def get_sample(self): + return self.sampler.getSample() def flatten_sample(self, sample): return self.sampler.space.flatten(sample) @@ -193,13 +195,14 @@ def evaluate_sample(self, sample): def run_server(self): start = time.time() - sample = self.get_sample(self.lastValue) + sample = self.get_sample() after_sampling = time.time() - self.lastValue = self.evaluate_sample(sample) + self.lastValue = self.evaluate_sample(sample.staticSample) + completed_sample = sample.complete(self.lastValue) after_simulation = time.time() timings = ServerTimings(sample_time=(after_sampling - start), simulate_time=(after_simulation - after_sampling)) - return sample, self.lastValue, timings + return completed_sample, self.lastValue, timings try: import ray diff --git a/tests/scenic/scenic_driving_behavior.scenic b/tests/scenic/scenic_driving_behavior.scenic new file mode 100644 index 0000000..1990dba --- /dev/null +++ b/tests/scenic/scenic_driving_behavior.scenic @@ -0,0 +1,17 @@ +param map = localPath('Town01.xodr') +model scenic.simulators.newtonian.driving_model + +foo = TimeSeries(VerifaiRange(0,0.01)) + +behavior TestBehavior(): + lastVal = None + while True: + newVal = foo.getSample() + assert newVal != lastVal, (newVal, lastVal) + lastVal = newVal + take SetThrottleAction(newVal) + +ego = new Car on road, with behavior TestBehavior() +new Car behind ego by VerifaiRange(1,4) + +terminate after 5 seconds diff --git a/tests/scenic/test_scenic.py b/tests/scenic/test_scenic.py index 2b3cf78..a6e3b9c 100644 --- a/tests/scenic/test_scenic.py +++ b/tests/scenic/test_scenic.py @@ -15,7 +15,7 @@ def test_objects(new_Object): f'ego = {new_Object} at 4 @ 9', maxIterations=1 ) - sample = sampler.nextSample() + sample = sampler.getSample() objects = sample.objects assert len(objects) == 1 pos = objects.object0.position @@ -28,7 +28,7 @@ def test_params(new_Object): f'ego = {new_Object}', maxIterations=1 ) - sample = sampler.nextSample() + sample = sampler.getSample() x = sample.params.x assert type(x) is float assert 3 <= x <= 5 @@ -39,7 +39,7 @@ def test_quoted_param(new_Object): f'ego = {new_Object}', maxIterations=1 ) - sample = sampler.nextSample() + sample = sampler.getSample() v = sampler.paramDictForSample(sample)['x/y'] assert type(v) is float assert 3 <= v <= 5 @@ -49,7 +49,7 @@ def test_lists(new_Object): f'ego = {new_Object} with foo [1, -1, 3.3]', maxIterations=1 ) - sample = sampler.nextSample() + sample = sampler.getSample() foo = sample.objects.object0.foo assert type(foo) is tuple assert foo == pytest.approx((1, -1, 3.3)) @@ -68,7 +68,7 @@ def test_object_order(new_Object): f' {new_Object} at 2*i @ 0', maxIterations=1 ) - sample = sampler.nextSample() + sample = sampler.getSample().complete(None) objects = sample.objects assert len(objects) == 11 for i in range(len(objects)): @@ -112,7 +112,7 @@ def test_active_save_restore(new_Object, tmpdir): def runSampler(sampler): for i in range(3): - sample = sampler.nextSample() + sample = sampler.getSample() print(f'Sample #{i}:') print(sample) @@ -179,3 +179,55 @@ def test_driving_dynamic(pathToLocalFile): server_class=ScenicServer, server_options=server_options) falsifier.run_falsifier() + +def test_driving_dynamic_behavior(pathToLocalFile): + path = pathToLocalFile('scenic_driving_behavior.scenic') + sampler = ScenicSampler.fromScenario( + path, + params=dict(render=False), + mode2D=True, + maxSteps=2 + ) + falsifier_params = DotMap( + n_iters=3, + save_error_table=False, + save_safe_table=False, + ) + server_options = DotMap(maxSteps=2, verbosity=3) + falsifier = generic_falsifier(sampler=sampler, + falsifier_params=falsifier_params, + server_class=ScenicServer, + server_options=server_options) + falsifier.run_falsifier() + +double_access_scenario = """ +model scenic.simulators.newtonian.model +foo = TimeSeries(VerifaiRange(0, 0.01)) +behavior TestBehavior(): + while True: + foo.getSample() + foo.getSample() + wait +ego = new Object with behavior TestBehavior() +""" + +def test_double_time_series_access(): + with pytest.raises(RuntimeError, match=r"Attempted `getSample` for a TimeSeries external parameter twice in one timestep."): + sampler = ScenicSampler.fromScenicCode( + double_access_scenario, + model='scenic.simulators.newtonian.model', + maxIterations=1, + maxSteps=2, + params=dict(render=False), + ) + falsifier_params = DotMap( + n_iters=3, + save_error_table=False, + save_safe_table=False, + ) + server_options = DotMap(maxSteps=2, verbosity=3) + falsifier = generic_falsifier(sampler=sampler, + falsifier_params=falsifier_params, + server_class=ScenicServer, + server_options=server_options) + falsifier.run_falsifier() diff --git a/tests/test_examples.py b/tests/test_examples.py index 764ae04..673252d 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -15,7 +15,7 @@ def test_example(): sampler = FeatureSampler.samplerFor(space) for i in range(3): - sample = sampler.nextSample() + sample = sampler.getSample().complete(None) print(f'Sample #{i}:') print(sample) flat = space.flatten(sample) diff --git a/tests/test_features.py b/tests/test_features.py index 3722ecc..ab02d3d 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -10,7 +10,7 @@ def test_fs_flatten(): }) sampler = FeatureSampler.randomSamplerFor(space) for i in range(100): - point = sampler.nextSample() + point = sampler.getSample().complete(None) flat = space.flatten(point) assert type(flat) is tuple assert len(flat) <= space.fixedFlattenedDimension @@ -25,7 +25,7 @@ def test_fs_flatten_fixed_dimension(): assert space.fixedFlattenedDimension == 4 sampler = FeatureSampler.randomSamplerFor(space) for i in range(100): - point = sampler.nextSample() + point = sampler.getSample().complete(None) flat = space.flatten(point, fixedDimension=True) assert type(flat) is tuple assert len(flat) == 4 @@ -58,7 +58,7 @@ def test_fs_flatten_fixed_dimension2(): assert space.fixedFlattenedDimension == 6 sampler = FeatureSampler.randomSamplerFor(space) for i in range(100): - point = sampler.nextSample() + point = sampler.getSample().complete(None) flat = space.flatten(point, fixedDimension=True) assert type(flat) is tuple assert len(flat) == 6 @@ -87,12 +87,73 @@ def test_fs_flatten_fixed_dimension2(): assert not any(space.coordinateIsNumerical(i) for i in range(1, 4)) assert all(space.coordinateIsNumerical(i) for i in range(4, 6)) +def test_fs_flatten_fixed_dimension_dynamic(): + space = FeatureSpace({ + 'a': Feature(DiscreteBox([0, 12])), + 'b': TimeSeriesFeature(Box((0, 1)), lengthDomain=DiscreteBox((0, 2))) + }, + timeBound=5 + ) + assert space.fixedFlattenedDimension == 13 + sampler = FeatureSampler.randomSamplerFor(space) + for i in range(100): + point = sampler.getSample() + duration = random.randint(0, 5) + for _ in range(duration): + point.getDynamicSample() + point = point.complete(None) + hash(point) + flat = space.flatten(point, fixedDimension=True) + assert type(flat) is tuple + assert len(flat) == 13 + assert eval(space.meaningOfFlatCoordinate(0)) == point.a[0] + assert flat[1] == duration + for t in range(duration): + offset = 2*t + bLen = len(point.dynamicSamples[t].b) + assert flat[2] == bLen + assert eval(space.meaningOfFlatCoordinate(2)) == bLen + if bLen < 1: + assert flat[offset+3] is None + else: + assert eval(space.meaningOfFlatCoordinate(offset+3)) == point.dynamicSamples[t].b[0][0] + if bLen < 2: + assert flat[offset+4] is None + else: + assert eval(space.meaningOfFlatCoordinate(offset+4)) == point.dynamicSamples[t].b[1][0] + unflat = space.unflatten(flat, fixedDimension=True) + assert point == unflat + assert space.pandasIndexForFlatCoordinate(0) == ('a', 0) + assert space.pandasIndexForFlatCoordinate(2) == ('b', 'length') + assert space.pandasIndexForFlatCoordinate(3) == ('b', 0, 0) + assert space.pandasIndexForFlatCoordinate(4) == ('b', 1, 0) + assert all(space.coordinateIsNumerical(i) for i in range(4)) + +def test_fs_utilities(): + space = FeatureSpace({ + 'a': Feature(DiscreteBox([0, 12])), + 'b': TimeSeriesFeature(Box((0, 1)), lengthDomain=DiscreteBox((1, 2))) + }, + timeBound=5 + ) + assert space.fixedFlattenedDimension == 13 + sampler = FeatureSampler.randomSamplerFor(space) + for i in range(100): + point = sampler.getSample() + duration = random.randint(0, 5) + for d in range(duration): + point.getDynamicSample() + assert 0 <= point.b[d][0][0] <= 1 + point = point.complete(None) + hash(point) + assert all(0 <= point.b[i][0][0] <= 1 for i in range(duration)) + def test_fs_distance(): box = Box([0, 10]) space = FeatureSpace({ 'a': Feature(box), 'b': Feature(box) }) sampler = FeatureSampler.randomSamplerFor(space) - pointA = sampler.nextSample() - pointB = sampler.nextSample() + pointA = sampler.getSample().staticSample + pointB = sampler.getSample().staticSample assert pointA != pointB assert space.distance(pointA, pointA) == 0 assert space.distance(pointB, pointB) == 0 diff --git a/tests/test_grid.py b/tests/test_grid.py index de16685..88ad57e 100644 --- a/tests/test_grid.py +++ b/tests/test_grid.py @@ -14,7 +14,8 @@ def test_grid(): dict_samples = defaultdict(int) while True: try: - sample = sampler.nextSample() + sample = sampler.getSample() + sample.complete(None) dict_samples[(sample.weather[0], sample.car_positions[0], sample.car_positions[1])] = 0 except TerminationException: @@ -46,7 +47,8 @@ def f(sample): y_samples = [] for i in range(21): - sample = sampler.nextSample() + sample = sampler.getSample() + sample.complete(None) samples.append(sample) y_samples.append(f(sample)) diff --git a/tests/test_halton.py b/tests/test_halton.py index 4c3cb7f..f8c73ff 100644 --- a/tests/test_halton.py +++ b/tests/test_halton.py @@ -24,7 +24,7 @@ def test_halton(): for i in range(3): print(f'Sample #{i}:') - print(sampler.nextSample()) + print(sampler.getSample().complete(None)) def test_save_restore(tmpdir): space = FeatureSpace({ diff --git a/tests/test_samplers.py b/tests/test_samplers.py index 4389e80..4b85802 100644 --- a/tests/test_samplers.py +++ b/tests/test_samplers.py @@ -3,9 +3,40 @@ import os.path from verifai.features import (Struct, Array, Box, DiscreteBox, - Feature, FeatureSpace) + Feature, TimeSeriesFeature, FeatureSpace) from verifai.samplers import RandomSampler, FeatureSampler +def test_feature_sampling(): + space = FeatureSpace({ + 'a': Feature(DiscreteBox([0, 12])), + 'b': Feature(Box((0, 1)), lengthDomain=DiscreteBox((0, 2))), + 'c': TimeSeriesFeature(Box((2,5))), + 'd': TimeSeriesFeature(Box((5,6)), lengthDomain=DiscreteBox((0,2))) + }, timeBound=10) + sampler = FeatureSampler.randomSamplerFor(space) + + sample = sampler.getSample() + static_point = sample + + assert len(static_point.a) == 1 + assert 0 <= static_point.a[0] <= 12 + assert 0 <= len(static_point.b) <= 2 + assert all(0 <= v[0] <= 1 for v in static_point.b) + + for _ in range(space.timeBound): + dynamic_point = sample.getDynamicSample() + + assert len(dynamic_point.c) == 1 + assert 2 <= dynamic_point.c[0] <= 5 + assert 0 <= len(dynamic_point.d) <= 2 + assert all(5 <= v[0] <= 6 for v in dynamic_point.d) + + for i in range(space.timeBound): + assert len(sample.c[i]) == 1 + assert 2 <= sample.c[i][0] <= 5 + assert 0 <= len(sample.d[i]) <= 2 + assert all(5 <= v[0] <= 6 for v in sample.d[i]) + ## Random sampling def test_domain_random(): @@ -36,7 +67,7 @@ def check(samples): assert any(sample[0][0].position[0] < sample[1][1].position[0] for sample in samples) - check([sampler.nextSample() for i in range(100)]) + check([sampler.getSample()[0] for _ in range(100)]) check(list(itertools.islice(sampler, 100))) def test_space_random(): @@ -48,7 +79,6 @@ def test_space_random(): def check(samples): for sample in samples: - assert type(sample) is space.makePoint a = sample.a assert type(a) is tuple assert len(a) == 1 @@ -72,7 +102,7 @@ def check(samples): assert any(len(sample.b) == 1 for sample in samples) assert any(len(sample.b) == 2 for sample in samples) - check([sampler.nextSample() for i in range(100)]) + check([sampler.getSample() for i in range(100)]) check(list(itertools.islice(sampler, 100))) def test_random_restore(tmpdir): @@ -84,7 +114,7 @@ def test_random_restore(tmpdir): path = os.path.join(tmpdir, 'blah.dat') sampler.saveToFile(path) - sample1 = sampler.nextSample() + sample1 = sampler.getSample().complete(0) sampler = FeatureSampler.restoreFromFile(path) - sample2 = sampler.nextSample() + sample2 = sampler.getSample().complete(0) assert sample1 == sample2 diff --git a/tests/utils.py b/tests/utils.py index 5b5d76f..316c89d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,11 +4,11 @@ from verifai.samplers import FeatureSampler def sampleWithFeedback(sampler, num_samples, f): - feedback = None samples = [] for i in range(num_samples): - sample = sampler.nextSample(feedback) + sample = sampler.getSample() feedback = f(sample) + sample.complete(feedback) print(f'Sample #{i}:') print(sample) samples.append(sample) @@ -19,18 +19,17 @@ def checkSaveRestore(sampler, tmpdir, iterations=1): feedback = None for i in range(iterations): sampler.saveToFile(path) - sample1 = sampler.nextSample(feedback) - sample2 = sampler.nextSample(-1) + sample1 = sampler.getSample().complete(-1) + sample2 = sampler.getSample().complete(0) sampler = FeatureSampler.restoreFromFile(path) - sample1b = sampler.nextSample(feedback) - sample2b = sampler.nextSample(-1) + sample1b = sampler.getSample().complete(-1) + sample2b = sampler.getSample().complete(1) assert sample1 != sample2 assert sample1 == sample1b assert sample2 == sample2b sampler.saveToFile(path) - sample3 = sampler.nextSample(1) + sample3 = sampler.getSample().complete(0) sampler = FeatureSampler.restoreFromFile(path) - sample3b = sampler.nextSample(1) + sample3b = sampler.getSample().complete(1) assert sample3 not in (sample1, sample2) assert sample3 == sample3b - feedback = 1