Issues registering custom prototype and custom miner

account_admin — Fri, 22 Mar 2019 17:52:32 GMT

# /opt/minemeld/engine/core/minemeld/nslookupm.py

from __future__ import absolute_import

import logging
#import requests
import socket

#from . import basepoller

LOG = logging.getLogger(__name__)


class NSLOOKUPM():
    def configure(self):
        # super(nsLookupM, self).configure()

        # self.polling_timeout = self.config.get('polling_timeout', 20)
        # self.verify_cert = self.config.get('verify_cert', True)

        # self.channel_name = self.config.get('channel_name', None)
        # if self.channel_name is None:
        #     raise ValueError('%s - channel name is required' % self.name)
        # self.url = 'https://www.youtube.com/user/{}/videos'.format{
        #    self.channel_name
        # }
        self.domain = 'dhswcms.upload.akamai.com'

        #self._build_iterator

    def _build_iterator(self):
        # called at every polling interval
        # here you should retrieve and return the list of item
        ip_list = socket.gethostbyname_ex(self.domain)[2]
        print(ip_list)
        return ip_list 

    def _process_item(self):
        # called on each item returned by _build_iterator
        # it should return a list of (indicator, value) pairs
        indicator_list = []
        for ipaddr in self._build_iterator():
            value = {
                'type': 'IP',
                'confidence': 100
            }
            indicator_list.append([ipaddr, value])
        return indicator_list 

# For testing purposes
nsLookup = NSLOOKUPM() # must configure first
nsLookup.configure() # configure
print(nsLookup._process_item()) #  process step




#test = nsLookupM()
#test()
#test._process_item()
#test._build_iterator()

I have gone through the steps to create a custom prototype and I have tried to make sure and followed the advice on these forums and the Youtube tutorial. Above is my customer prototype that I placed in the ft folder.

Below is the nodes.json file in the core folder:

{
    "minemeld.ft.anomali.Intelligence": {
        "class": "minemeld.ft.anomali:Intelligence"
    },
    "minemeld.ft.nslookupm.NSLOOKUPM":{
        "class": "minemeld.ft.nslookupm:NSLOOKUPM"
    }
    "minemeld.ft.auscert.MaliciousURLFeed": {
        "class": "minemeld.ft.auscert:MaliciousURLFeed"
    },
    "minemeld.ft.autofocus.ExportList": {
        "class": "minemeld.ft.autofocus:ExportList"
    },
    "minemeld.ft.azure.AzureXML": {
        "class": "minemeld.ft.azure:AzureXML"
    },
    "minemeld.ft.cif.Feed": {
        "class": "minemeld.ft.cif:Feed"
    },
    "minemeld.ft.ciscoise.ErsSgt": {
        "class": "minemeld.ft.ciscoise:ErsSgt"
    },
    "minemeld.ft.csv.CSVFT": {
        "class": "minemeld.ft.csv:CSVFT"
    },
    "minemeld.ft.dag.DagPusher": {
        "class": "minemeld.ft.dag:DagPusher"
    },
    "minemeld.ft.google.GoogleNetBlocks": {
        "class": "minemeld.ft.google:GoogleNetBlocks"
    },
    "minemeld.ft.google.GoogleCloudNetBlocks": {
        "class": "minemeld.ft.google:GoogleCloudNetBlocks"
    },
    "minemeld.ft.google.GoogleSPF": {
        "class": "minemeld.ft.google:GoogleSPF"
    },
    "minemeld.ft.http.HttpFT": {
        "class": "minemeld.ft.http:HttpFT"
    },
    "minemeld.ft.ipop.AggregateIPv4FT": {
        "class": "minemeld.ft.ipop:AggregateIPv4FT"
    },
    "minemeld.ft.json.SimpleJSON": {
        "class": "minemeld.ft.json:SimpleJSON"
    },
    "minemeld.ft.local.YamlFT": {
        "class": "minemeld.ft.local:YamlFT"
    },
    "minemeld.ft.local.YamlIPv4FT": {
        "class": "minemeld.ft.local:YamlIPv4FT"
    },
    "minemeld.ft.local.YamlURLFT": {
        "class": "minemeld.ft.local:YamlURLFT"
    },
    "minemeld.ft.local.YamlDomainFT": {
        "class": "minemeld.ft.local:YamlDomainFT"
    },
    "minemeld.ft.local.YamlIPv6FT": {
        "class": "minemeld.ft.local:YamlIPv6FT"
    },
    "minemeld.ft.logstash.LogstashOutput": {
        "class": "minemeld.ft.logstash:LogstashOutput"
    },
    "minemeld.ft.o365.O365XML": {
        "class": "minemeld.ft.o365:O365XML"
    },
    "minemeld.ft.o365.O365API": {
        "class": "minemeld.ft.o365:O365API"
    },
    "minemeld.ft.op.AggregateFT": {
        "class": "minemeld.ft.op:AggregateFT"
    },
    "minemeld.ft.phishme.Intelligence": {
        "class": "minemeld.ft.phishme:Intelligence"
    },
    "minemeld.ft.proofpoint.ETIntelligence": {
        "class": "minemeld.ft.proofpoint:ETIntelligence"
    },
    "minemeld.ft.proofpoint.EmergingThreatsIP": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsIP"
    },
    "minemeld.ft.proofpoint.EmergingThreatsDomain": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsDomain"
    },
    "minemeld.ft.recordedfuture.IPRiskList": {
        "class": "minemeld.ft.recordedfuture:IPRiskList"
    },
	"minemeld.ft.recordedfuture.DomainRiskList": {
        "class": "minemeld.ft.recordedfuture:DomainRiskList"
    },
    "minemeld.ft.redis.RedisSet": {
        "class": "minemeld.ft.redis:RedisSet"
    },
    "minemeld.ft.syslog.SyslogMatcher": {
        "class": "minemeld.ft.syslog:SyslogMatcher"
    },
    "minemeld.ft.syslog.SyslogMiner": {
        "class": "minemeld.ft.syslog:SyslogMiner"
    },
    "minemeld.ft.taxii.TaxiiClient": {
        "class": "minemeld.ft.taxii:TaxiiClient"
    },
    "minemeld.ft.taxii.DataFeed": {
        "class": "minemeld.ft.taxii:DataFeed"
    },
    "minemeld.ft.threatq.Export": {
        "class": "minemeld.ft.threatq:Export"
    },
    "minemeld.ft.tmt.DTIAPI": {
        "class": "minemeld.ft.tmt:DTIAPI"
    },
    "minemeld.ft.vt.Notifications": {
        "class": "minemeld.ft.vt:Notifications"
    },
    "minemeld.ft.mm.JSONSEQMiner": {
        "class": "minemeld.ft.mm:JSONSEQMiner"
    },
    "minemeld.ft.localdb.Miner": {
        "class": "minemeld.ft.localdb:Miner"
    },
    "minemeld.ft.threatconnect.IndicatorsMiner": {
        "class": "minemeld.ft.threatconnect:IndicatorsMiner"
    },
    "minemeld.ft.threatconnect.GroupsMiner": {
        "class": "minemeld.ft.threatconnect:GroupsMiner"
    },
    "minemeld.ft.visa.VTI": {
        "class": "minemeld.ft.visa:VTI"
    }
}

This is the .yml file I placed in the prototypes fodler:

{
    "minemeld.ft.anomali.Intelligence": {
        "class": "minemeld.ft.anomali:Intelligence"
    },
    "minemeld.ft.nslookupm.NSLOOKUPM":{
        "class": "minemeld.ft.nslookupm:NSLOOKUPM"
    }
    "minemeld.ft.auscert.MaliciousURLFeed": {
        "class": "minemeld.ft.auscert:MaliciousURLFeed"
    },
    "minemeld.ft.autofocus.ExportList": {
        "class": "minemeld.ft.autofocus:ExportList"
    },
    "minemeld.ft.azure.AzureXML": {
        "class": "minemeld.ft.azure:AzureXML"
    },
    "minemeld.ft.cif.Feed": {
        "class": "minemeld.ft.cif:Feed"
    },
    "minemeld.ft.ciscoise.ErsSgt": {
        "class": "minemeld.ft.ciscoise:ErsSgt"
    },
    "minemeld.ft.csv.CSVFT": {
        "class": "minemeld.ft.csv:CSVFT"
    },
    "minemeld.ft.dag.DagPusher": {
        "class": "minemeld.ft.dag:DagPusher"
    },
    "minemeld.ft.google.GoogleNetBlocks": {
        "class": "minemeld.ft.google:GoogleNetBlocks"
    },
    "minemeld.ft.google.GoogleCloudNetBlocks": {
        "class": "minemeld.ft.google:GoogleCloudNetBlocks"
    },
    "minemeld.ft.google.GoogleSPF": {
        "class": "minemeld.ft.google:GoogleSPF"
    },
    "minemeld.ft.http.HttpFT": {
        "class": "minemeld.ft.http:HttpFT"
    },
    "minemeld.ft.ipop.AggregateIPv4FT": {
        "class": "minemeld.ft.ipop:AggregateIPv4FT"
    },
    "minemeld.ft.json.SimpleJSON": {
        "class": "minemeld.ft.json:SimpleJSON"
    },
    "minemeld.ft.local.YamlFT": {
        "class": "minemeld.ft.local:YamlFT"
    },
    "minemeld.ft.local.YamlIPv4FT": {
        "class": "minemeld.ft.local:YamlIPv4FT"
    },
    "minemeld.ft.local.YamlURLFT": {
        "class": "minemeld.ft.local:YamlURLFT"
    },
    "minemeld.ft.local.YamlDomainFT": {
        "class": "minemeld.ft.local:YamlDomainFT"
    },
    "minemeld.ft.local.YamlIPv6FT": {
        "class": "minemeld.ft.local:YamlIPv6FT"
    },
    "minemeld.ft.logstash.LogstashOutput": {
        "class": "minemeld.ft.logstash:LogstashOutput"
    },
    "minemeld.ft.o365.O365XML": {
        "class": "minemeld.ft.o365:O365XML"
    },
    "minemeld.ft.o365.O365API": {
        "class": "minemeld.ft.o365:O365API"
    },
    "minemeld.ft.op.AggregateFT": {
        "class": "minemeld.ft.op:AggregateFT"
    },
    "minemeld.ft.phishme.Intelligence": {
        "class": "minemeld.ft.phishme:Intelligence"
    },
    "minemeld.ft.proofpoint.ETIntelligence": {
        "class": "minemeld.ft.proofpoint:ETIntelligence"
    },
    "minemeld.ft.proofpoint.EmergingThreatsIP": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsIP"
    },
    "minemeld.ft.proofpoint.EmergingThreatsDomain": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsDomain"
    },
    "minemeld.ft.recordedfuture.IPRiskList": {
        "class": "minemeld.ft.recordedfuture:IPRiskList"
    },
	"minemeld.ft.recordedfuture.DomainRiskList": {
        "class": "minemeld.ft.recordedfuture:DomainRiskList"
    },
    "minemeld.ft.redis.RedisSet": {
        "class": "minemeld.ft.redis:RedisSet"
    },
    "minemeld.ft.syslog.SyslogMatcher": {
        "class": "minemeld.ft.syslog:SyslogMatcher"
    },
    "minemeld.ft.syslog.SyslogMiner": {
        "class": "minemeld.ft.syslog:SyslogMiner"
    },
    "minemeld.ft.taxii.TaxiiClient": {
        "class": "minemeld.ft.taxii:TaxiiClient"
    },
    "minemeld.ft.taxii.DataFeed": {
        "class": "minemeld.ft.taxii:DataFeed"
    },
    "minemeld.ft.threatq.Export": {
        "class": "minemeld.ft.threatq:Export"
    },
    "minemeld.ft.tmt.DTIAPI": {
        "class": "minemeld.ft.tmt:DTIAPI"
    },
    "minemeld.ft.vt.Notifications": {
        "class": "minemeld.ft.vt:Notifications"
    },
    "minemeld.ft.mm.JSONSEQMiner": {
        "class": "minemeld.ft.mm:JSONSEQMiner"
    },
    "minemeld.ft.localdb.Miner": {
        "class": "minemeld.ft.localdb:Miner"
    },
    "minemeld.ft.threatconnect.IndicatorsMiner": {
        "class": "minemeld.ft.threatconnect:IndicatorsMiner"
    },
    "minemeld.ft.threatconnect.GroupsMiner": {
        "class": "minemeld.ft.threatconnect:GroupsMiner"
    },
    "minemeld.ft.visa.VTI": {
        "class": "minemeld.ft.visa:VTI"
    }
}

This is the minemeldlocal.yml file:

{
    "minemeld.ft.anomali.Intelligence": {
        "class": "minemeld.ft.anomali:Intelligence"
    },
    "minemeld.ft.nslookupm.NSLOOKUPM":{
        "class": "minemeld.ft.nslookupm:NSLOOKUPM"
    }
    "minemeld.ft.auscert.MaliciousURLFeed": {
        "class": "minemeld.ft.auscert:MaliciousURLFeed"
    },
    "minemeld.ft.autofocus.ExportList": {
        "class": "minemeld.ft.autofocus:ExportList"
    },
    "minemeld.ft.azure.AzureXML": {
        "class": "minemeld.ft.azure:AzureXML"
    },
    "minemeld.ft.cif.Feed": {
        "class": "minemeld.ft.cif:Feed"
    },
    "minemeld.ft.ciscoise.ErsSgt": {
        "class": "minemeld.ft.ciscoise:ErsSgt"
    },
    "minemeld.ft.csv.CSVFT": {
        "class": "minemeld.ft.csv:CSVFT"
    },
    "minemeld.ft.dag.DagPusher": {
        "class": "minemeld.ft.dag:DagPusher"
    },
    "minemeld.ft.google.GoogleNetBlocks": {
        "class": "minemeld.ft.google:GoogleNetBlocks"
    },
    "minemeld.ft.google.GoogleCloudNetBlocks": {
        "class": "minemeld.ft.google:GoogleCloudNetBlocks"
    },
    "minemeld.ft.google.GoogleSPF": {
        "class": "minemeld.ft.google:GoogleSPF"
    },
    "minemeld.ft.http.HttpFT": {
        "class": "minemeld.ft.http:HttpFT"
    },
    "minemeld.ft.ipop.AggregateIPv4FT": {
        "class": "minemeld.ft.ipop:AggregateIPv4FT"
    },
    "minemeld.ft.json.SimpleJSON": {
        "class": "minemeld.ft.json:SimpleJSON"
    },
    "minemeld.ft.local.YamlFT": {
        "class": "minemeld.ft.local:YamlFT"
    },
    "minemeld.ft.local.YamlIPv4FT": {
        "class": "minemeld.ft.local:YamlIPv4FT"
    },
    "minemeld.ft.local.YamlURLFT": {
        "class": "minemeld.ft.local:YamlURLFT"
    },
    "minemeld.ft.local.YamlDomainFT": {
        "class": "minemeld.ft.local:YamlDomainFT"
    },
    "minemeld.ft.local.YamlIPv6FT": {
        "class": "minemeld.ft.local:YamlIPv6FT"
    },
    "minemeld.ft.logstash.LogstashOutput": {
        "class": "minemeld.ft.logstash:LogstashOutput"
    },
    "minemeld.ft.o365.O365XML": {
        "class": "minemeld.ft.o365:O365XML"
    },
    "minemeld.ft.o365.O365API": {
        "class": "minemeld.ft.o365:O365API"
    },
    "minemeld.ft.op.AggregateFT": {
        "class": "minemeld.ft.op:AggregateFT"
    },
    "minemeld.ft.phishme.Intelligence": {
        "class": "minemeld.ft.phishme:Intelligence"
    },
    "minemeld.ft.proofpoint.ETIntelligence": {
        "class": "minemeld.ft.proofpoint:ETIntelligence"
    },
    "minemeld.ft.proofpoint.EmergingThreatsIP": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsIP"
    },
    "minemeld.ft.proofpoint.EmergingThreatsDomain": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsDomain"
    },
    "minemeld.ft.recordedfuture.IPRiskList": {
        "class": "minemeld.ft.recordedfuture:IPRiskList"
    },
	"minemeld.ft.recordedfuture.DomainRiskList": {
        "class": "minemeld.ft.recordedfuture:DomainRiskList"
    },
    "minemeld.ft.redis.RedisSet": {
        "class": "minemeld.ft.redis:RedisSet"
    },
    "minemeld.ft.syslog.SyslogMatcher": {
        "class": "minemeld.ft.syslog:SyslogMatcher"
    },
    "minemeld.ft.syslog.SyslogMiner": {
        "class": "minemeld.ft.syslog:SyslogMiner"
    },
    "minemeld.ft.taxii.TaxiiClient": {
        "class": "minemeld.ft.taxii:TaxiiClient"
    },
    "minemeld.ft.taxii.DataFeed": {
        "class": "minemeld.ft.taxii:DataFeed"
    },
    "minemeld.ft.threatq.Export": {
        "class": "minemeld.ft.threatq:Export"
    },
    "minemeld.ft.tmt.DTIAPI": {
        "class": "minemeld.ft.tmt:DTIAPI"
    },
    "minemeld.ft.vt.Notifications": {
        "class": "minemeld.ft.vt:Notifications"
    },
    "minemeld.ft.mm.JSONSEQMiner": {
        "class": "minemeld.ft.mm:JSONSEQMiner"
    },
    "minemeld.ft.localdb.Miner": {
        "class": "minemeld.ft.localdb:Miner"
    },
    "minemeld.ft.threatconnect.IndicatorsMiner": {
        "class": "minemeld.ft.threatconnect:IndicatorsMiner"
    },
    "minemeld.ft.threatconnect.GroupsMiner": {
        "class": "minemeld.ft.threatconnect:GroupsMiner"
    },
    "minemeld.ft.visa.VTI": {
        "class": "minemeld.ft.visa:VTI"
    }
}

Re: Issues registering custom prototype and custom miner

account_admin — Fri, 22 Mar 2019 17:53:45 GMT

The error I am getting specifically is that I am trying to commit the nslookupm prototype but I am getting the following error:

COMMIT FAILED: Unknown node class minemeld.ft.nslookupm.NSLOOKUPM in node-1553265796327

topic Re: Issues registering custom prototype and custom miner in Automation/API Discussions

Issues registering custom prototype and custom miner

Re: Issues registering custom prototype and custom miner