- Access exclusive content
- Connect with peers
- Share your expertise
- Find support resources
06-12-2020 06:46 AM
The error started coming up after I created and committed a custom miner. Pasting what looks like the relevant portion here:
2020-06-09T10:24:25 (26502)launcher.main INFO: mm-run.py config: _Config(nodes={'GridMeld': {'inputs': [], 'config': {'attributes': {'confidence': 100, 'share_level': 'red'}, 'interval': 3600}, 'class': 'minemeld.ft.localdb.Miner', 'output': True}, 'o365-any-any-url-feed': {'inputs': ['o365-url-processor'], 'config': {'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ['confidence > 75', "share_level == 'green'"], 'name': 'accept confidence > 75 and share level green', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.redis.RedisSet', 'output': False}, 'spamhaus_processor': {'inputs': ['spamhaus'], 'config': {'whitelist_prefixes': ['wl'], 'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ["type == 'IPv4'"], 'name': 'accept IPv4', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.ipop.AggregateIPv4FT', 'output': True}, 'o365-usgovdod-any-miner': {'inputs': [], 'config': {'attributes': {'confidence': 100, 'share_level': 'green'}, 'service_areas': None, 'age_out': {'default': None, 'sudden_death': True, 'interval': 1800}, 'instance': 'USGovDoD'}, 'class': 'minemeld.ft.o365.O365API', 'output': True}, 'freeBaseOffering': {'inputs': ['myBaseOffering'], 'config': {'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ['confidence > 75', "share_level == 'green'"], 'name': 'accept confidence > 75 and share level green', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.redis.RedisSet', 'output': False}, 'spamhaus_DROP': {'output': True, 'config': {'indicator': {'regex': '^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}/[0-9]{1,2}'}, 'source_name': 'spamhaus.DROP', 'age_out': {'default': None, 'sudden_death': True, 'interval': 677}, 'url': 'https://www.spamhaus.org/drop/drop.txt', 'attributes': {'direction': 'inbound', 'type': 'IPv4', 'confidence': 100, 'share_level': 'green'}, 'ignore_regex': '^;.*'}, 'class': 'minemeld.ft.http.HttpFT'}, 'o365-worldwide-any-miner': {'inputs': [], 'config': {'attributes': {'confidence': 100, 'share_level': 'green'}, 'service_areas': None, 'age_out': {'default': None, 'sudden_death': True, 'interval': 1800}, 'instance': 'Worldwide'}, 'class': 'minemeld.ft.o365.O365API', 'output': True}, 'aws-ip-miner': {'inputs': [], 'config': {'extractor': "prefixes[?service=='AMAZON']", 'indicator': 'ip_prefix', 'source_name': 'aws.AMAZON', 'age_out': {'default': None, 'sudden_death': True, 'interval': 257}, 'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json', 'fields': ['region', 'service'], 'prefix': 'aws', 'attributes': {'confidence': 100, 'type': 'IPv4', 'share_level': 'green'}}, 'class': 'minemeld.ft.json.SimpleJSON', 'output': True}, 'bluejeans': {'inputs': [], 'config': {'url': 'https://support.bluejeans.com/s/article/TCP-UDP-ports-used-by-BlueJeans-Network', 'attributes': {'confidence': 100, 'type': 'IPv4', 'share_level': 'green'}, 'indicator': 'regex:^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\/\\d\\d$', 'source_name': 'bluejeans', 'age_out': {'default': None, 'sudden_death': True, 'interval': 257}}, 'class': 'minemeld.ft.http.HttpFT', 'output': True}, 'spamhaus_EDROP': {'output': True, 'config': {'indicator': {'regex': '^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}/[0-9]{1,2}'}, 'source_name': 'spamhaus.EDROP', 'age_out': {'default': None, 'sudden_death': True, 'interval': 677}, 'url': 'https://www.spamhaus.org/drop/edrop.txt', 'attributes': {'direction': 'inbound', 'type': 'IPv4', 'confidence': 100, 'share_level': 'green'}, 'ignore_regex': '^;.*'}, 'class': 'minemeld.ft.http.HttpFT'}, 'o365_filter_ipv4': {'inputs': ['o365_germany_any', 'cloudflare-m2', 'spamhaus'], 'indicator_types': ['IPv4'], 'node_type': 'processor', 'output': True, 'config': {'whitelist_prefixes': ['wl'], 'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ["type == 'IPv4'"], 'name': 'accept IPv4', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.ipop.AggregateIPv4FT'}, 'DAG-Pusher': {'inputs': ['GridMeld'], 'config': {}, 'class': 'minemeld.ft.dag_ng.DagPusher', 'output': False}, 'o365-any-any-ipv4-feed': {'inputs': ['o365-ipv4-processor'], 'config': {'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ['confidence > 75', "share_level == 'green'"], 'name': 'accept confidence > 75 and share level green', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.redis.RedisSet', 'output': False}, 'o365-usgovgcchigh-any-miner': {'inputs': [], 'config': {'attributes': {'confidence': 100, 'share_level': 'green'}, 'service_areas': None, 'age_out': {'default': None, 'sudden_death': True, 'interval': 1800}, 'instance': 'USGovGCCHigh'}, 'class': 'minemeld.ft.o365.O365API', 'output': True}, 'cloudflare-m2': {'inputs': [], 'config': {'url': 'https://www.cloudflare.com/ips-v4', 'attributes': {'confidence': 100, 'type': 'IPv4', 'share_level': 'green'}, 'source_name': 'cloudflare', 'age_out': {'default': None, 'sudden_death': True, 'interval': 257}}, 'class': 'minemeld.ft.http.HttpFT', 'output': True}, 'o1': {'inputs': ['o365_filter_ipv4'], 'config': {'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ['confidence > 75', "share_level == 'green'"], 'name': 'accept confidence > 75 and share level green', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.redis.RedisSet', 'output': False}, 'o365-any-any-ipv6-feed': {'inputs': ['o365-ipv6-processor'], 'config': {'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ['confidence > 75', "share_level == 'green'"], 'name': 'accept confidence > 75 and share level green', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.redis.RedisSet', 'output': False}, 'spamhaus': {'inputs': [], 'config': {'indicator': {'regex': '^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}/[0-9]{1,2}'}, 'source_name': 'spamhaus.DROP', 'age_out': {'default': None, 'sudden_death': True, 'interval': 677}, 'url': 'https://www.spamhaus.org/drop/drop.txt', 'attributes': {'direction': 'inbound', 'type': 'IPv4', 'confidence': 100, 'share_level': 'green'}, 'ignore_regex': '^;.*'}, 'class': 'minemeld.ft.http.HttpFT', 'output': True}, 'aws-ip-feed': {'inputs': ['aws-ip-processor'], 'config': {'store_value': True, 'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ['confidence > 75'], 'name': 'accept confidence > 75', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.redis.RedisSet', 'output': False}, 'o365-germany-any-miner': {'inputs': [], 'config': {'attributes': {'confidence': 100, 'share_level': 'green'}, 'service_areas': None, 'age_out': {'default': None, 'sudden_death': True, 'interval': 1800}, 'instance': 'Germany'}, 'class': 'minemeld.ft.o365.O365API', 'output': True}, 'dshield_blocklist': {'output': True, 'config': {'indicator': {'regex': '^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})\\t([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})', 'transform': '\\1-\\2'}, 'source_name': 'dshield.block', 'age_out': {'default': None, 'sudden_death': True, 'interval': 257}, 'url': 'https://www.dshield.org/block.txt', 'fields': {'dshield_name': {'regex': '^.*\\t.*\\t[0-9]+\\t[0-9]+\\t([^\\t]+)', 'transform': '\\1'}, 'dshield_country': {'regex': '^.*\\t.*\\t[0-9]+\\t[0-9]+\\t[^\\t]+\\t([A-Z]+)', 'transform': '\\1'}, 'dshield_nattacks': {'regex': '^.*\\t.*\\t[0-9]+\\t([0-9]+)', 'transform': '\\1'}, 'dshield_email': {'regex': '^.*\\t.*\\t[0-9]+\\t[0-9]+\\t[^\\t]+\\t[A-Z]+\\t(\\S+)', 'transform': '\\1'}}, 'interval': 619, 'attributes': {'direction': 'inbound', 'type': 'IPv4', 'confidence': 100, 'share_level': 'green'}, 'ignore_regex': '[#S].*'}, 'class': 'minemeld.ft.http.HttpFT'}, 'myBaseOffering': {'inputs': ['spamhaus_EDROP', 'spamhaus_DROP', 'dshield_blocklist'], 'config': {'whitelist_prefixes': ['wl'], 'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ["type == 'IPv4'"], 'name': 'accept IPv4', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.ipop.AggregateIPv4FT', 'output': True}, 'o365-ipv4-processor': {'inputs': ['o365-worldwide-any-miner', 'o365-germany-any-miner', 'o365-china-any-miner', 'o365-usgovgcchigh-any-miner', 'o365-usgovdod-any-miner'], 'config': {'whitelist_prefixes': ['wl'], 'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ["type == 'IPv4'"], 'name': 'accept IPv4', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.ipop.AggregateIPv4FT', 'output': True}, 'bluejeans-ipv4-feed': {'inputs': ['bluejeans-ipv4-processor'], 'config': {'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'actions': ['accept']}]}, 'class': 'minemeld.ft.redis.RedisSet', 'output': False}, 'aws-ip-processor': {'inputs': ['aws-ip-miner'], 'config': {'whitelist_prefixes': ['wl'], 'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ["type == 'IPv4'"], 'name': 'accept IPv4', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.ipop.AggregateIPv4FT', 'output': True}, 'o365-china-any-miner': {'inputs': [], 'config': {'attributes': {'confidence': 100, 'share_level': 'green'}, 'service_areas': None, 'age_out': {'default': None, 'sudden_death': True, 'interval': 1800}, 'instance': 'China'}, 'class': 'minemeld.ft.o365.O365API', 'output': True}, 'o365-china-ipv4-feed': {'inputs': ['o365-china-ipv4-processor'], 'config': {'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ['confidence > 75', "share_level == 'green'"], 'name': 'accept confidence > 75 and share level green', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.redis.RedisSet', 'output': False}, 'o365-china-ipv4-processor': {'inputs': ['o365-china-any-miner'], 'config': {'whitelist_prefixes': ['wl'], 'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ["type == 'IPv4'"], 'name': 'accept IPv4', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.ipop.AggregateIPv4FT', 'output': True}, 'o365_germany_any': {'inputs': [], 'config': {'attributes': {'confidence': 100, 'share_level': 'green'}, 'service_areas': None, 'age_out': {'default': None, 'sudden_death': True, 'interval': 1800}, 'instance': 'Germany'}, 'class': 'minemeld.ft.o365.O365API', 'output': True}, 'bluejeans-ipv4-processor': {'inputs': ['bluejeans'], 'config': {'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ["type == 'IPv4'"], 'name': 'accept IPv4', 'actions': ['accept']}]}, 'class': 'minemeld.ft.ipop.AggregateIPv4FT', 'output': True}, 'o365-url-processor': {'inputs': ['o365-worldwide-any-miner', 'o365-germany-any-miner', 'o365-china-any-miner', 'o365-usgovgcchigh-any-miner', 'o365-usgovdod-any-miner'], 'config': {'whitelist_prefixes': ['wl'], 'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ["type == 'URL'"], 'name': 'accept URL', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.op.AggregateFT', 'output': True}, 'o365-ipv6-processor': {'inputs': ['o365-worldwide-any-miner', 'o365-germany-any-miner', 'o365-china-any-miner', 'o365-usgovgcchigh-any-miner', 'o365-usgovdod-any-miner'], 'config': {'whitelist_prefixes': ['wl'], 'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ["type == 'IPv6'"], 'name': 'accept IPv6', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.op.AggregateFT', 'output': True}, 'spamhaus_output_ip': {'inputs': ['spamhaus_processor'], 'config': {'infilters': [{'conditions': ["__method == 'withdraw'"], 'name': 'accept withdraws', 'actions': ['accept']}, {'conditions': ['confidence > 75', "share_level == 'green'"], 'name': 'accept confidence > 75 and share level green', 'actions': ['accept']}, {'name': 'drop all', 'actions': ['drop']}]}, 'class': 'minemeld.ft.redis.RedisSet', 'output': False}}, fabric={'config': {'priority': -2, 'num_connections': 50}, 'class': 'ZMQRedis'}, mgmtbus={'slave': {}, 'master': {}, 'transport': {'config': {'priority': 2, 'num_connections': 10}, 'class': 'ZMQRedis'}}, changes=[])
This cascades down and leads to errors like :
2020-06-09T10:24:25 (26520)base.read_checkpoint ERROR: bluejeans-ipv4-processor - Error reading last checkpoint
Traceback (most recent call last):
File "/opt/minemeld/engine/current/lib/python2.7/site-packages/minemeld/ft/base.py", line 255, in read_checkpoint
with open(self.name+'.chkp', 'r') as f:
IOError: [Errno 2] No such file or directory: 'bluejeans-ipv4-processor.chkp'
2020-06-09T10:24:25 (26520)base.connect INFO: bluejeans-ipv4-processor - requesting fabric sub channel for bluejeans
The .chkp files it's looking for actually don't exist.
2020-06-09T10:28:11 (27805)launcher._run_chassis ERROR: Exception in chassis main procedure
Traceback (most recent call last):
File "/opt/minemeld/engine/current/lib/python2.7/site-packages/minemeld/run/launcher.py", line 52, in _run_chassis
c.configure(fts)
File "/opt/minemeld/engine/current/lib/python2.7/site-packages/minemeld/chassis.py", line 101, in configure
config=ftconfig.get('config', {})
File "/opt/minemeld/engine/current/lib/python2.7/site-packages/minemeld/ft/__init__.py", line 10, in factory
config=config
File "/opt/minemeld/engine/current/lib/python2.7/site-packages/minemeld/ft/basepoller.py", line 321, in __init__
super(BasePollerFT, self).__init__(name, chassis, config)
File "/opt/minemeld/engine/current/lib/python2.7/site-packages/minemeld/ft/base.py", line 199, in __init__
self.configure()
File "/opt/minemeld/engine/current/lib/python2.7/site-packages/minemeld/ft/http.py", line 120, in configure
self.indicator['regex'] = re.compile(self.indicator['regex'])
TypeError: string indices must be integers, not str
Any idea how I can revert to a prior working configuration?
Click Accept as Solution to acknowledge that the answer to your question has been provided.
The button appears next to the replies on topics you’ve started. The member who gave the solution and all future visitors to this topic will appreciate it!
These simple actions take just seconds of your time, but go a long way in showing appreciation for community members and the LIVEcommunity as a whole!
The LIVEcommunity thanks you for your participation!