Issues registering custom prototype and custom miner

Reply
L0 Member

Issues registering custom prototype and custom miner

  1. # /opt/minemeld/engine/core/minemeld/nslookupm.py
    
    from __future__ import absolute_import
    
    import logging
    #import requests
    import socket
    
    #from . import basepoller
    
    LOG = logging.getLogger(__name__)
    
    
    class NSLOOKUPM():
        def configure(self):
            # super(nsLookupM, self).configure()
    
            # self.polling_timeout = self.config.get('polling_timeout', 20)
            # self.verify_cert = self.config.get('verify_cert', True)
    
            # self.channel_name = self.config.get('channel_name', None)
            # if self.channel_name is None:
            #     raise ValueError('%s - channel name is required' % self.name)
            # self.url = 'https://www.youtube.com/user/{}/videos'.format{
            #    self.channel_name
            # }
            self.domain = 'dhswcms.upload.akamai.com'
    
            #self._build_iterator
    
        def _build_iterator(self):
            # called at every polling interval
            # here you should retrieve and return the list of item
            ip_list = socket.gethostbyname_ex(self.domain)[2]
            print(ip_list)
            return ip_list 
    
        def _process_item(self):
            # called on each item returned by _build_iterator
            # it should return a list of (indicator, value) pairs
            indicator_list = []
            for ipaddr in self._build_iterator():
                value = {
                    'type': 'IP',
                    'confidence': 100
                }
                indicator_list.append([ipaddr, value])
            return indicator_list 
    
    # For testing purposes
    nsLookup = NSLOOKUPM() # must configure first
    nsLookup.configure() # configure
    print(nsLookup._process_item()) #  process step
    
    
    
    
    #test = nsLookupM()
    #test()
    #test._process_item()
    #test._build_iterator()
    I have gone through the steps to create a custom prototype and I have tried to make sure and followed the advice on these forums and the Youtube tutorial. Above is my customer prototype that I placed in the ft folder.

Below is the  nodes.json file in the core folder:

 

{
    "minemeld.ft.anomali.Intelligence": {
        "class": "minemeld.ft.anomali:Intelligence"
    },
    "minemeld.ft.nslookupm.NSLOOKUPM":{
        "class": "minemeld.ft.nslookupm:NSLOOKUPM"
    }
    "minemeld.ft.auscert.MaliciousURLFeed": {
        "class": "minemeld.ft.auscert:MaliciousURLFeed"
    },
    "minemeld.ft.autofocus.ExportList": {
        "class": "minemeld.ft.autofocus:ExportList"
    },
    "minemeld.ft.azure.AzureXML": {
        "class": "minemeld.ft.azure:AzureXML"
    },
    "minemeld.ft.cif.Feed": {
        "class": "minemeld.ft.cif:Feed"
    },
    "minemeld.ft.ciscoise.ErsSgt": {
        "class": "minemeld.ft.ciscoise:ErsSgt"
    },
    "minemeld.ft.csv.CSVFT": {
        "class": "minemeld.ft.csv:CSVFT"
    },
    "minemeld.ft.dag.DagPusher": {
        "class": "minemeld.ft.dag:DagPusher"
    },
    "minemeld.ft.google.GoogleNetBlocks": {
        "class": "minemeld.ft.google:GoogleNetBlocks"
    },
    "minemeld.ft.google.GoogleCloudNetBlocks": {
        "class": "minemeld.ft.google:GoogleCloudNetBlocks"
    },
    "minemeld.ft.google.GoogleSPF": {
        "class": "minemeld.ft.google:GoogleSPF"
    },
    "minemeld.ft.http.HttpFT": {
        "class": "minemeld.ft.http:HttpFT"
    },
    "minemeld.ft.ipop.AggregateIPv4FT": {
        "class": "minemeld.ft.ipop:AggregateIPv4FT"
    },
    "minemeld.ft.json.SimpleJSON": {
        "class": "minemeld.ft.json:SimpleJSON"
    },
    "minemeld.ft.local.YamlFT": {
        "class": "minemeld.ft.local:YamlFT"
    },
    "minemeld.ft.local.YamlIPv4FT": {
        "class": "minemeld.ft.local:YamlIPv4FT"
    },
    "minemeld.ft.local.YamlURLFT": {
        "class": "minemeld.ft.local:YamlURLFT"
    },
    "minemeld.ft.local.YamlDomainFT": {
        "class": "minemeld.ft.local:YamlDomainFT"
    },
    "minemeld.ft.local.YamlIPv6FT": {
        "class": "minemeld.ft.local:YamlIPv6FT"
    },
    "minemeld.ft.logstash.LogstashOutput": {
        "class": "minemeld.ft.logstash:LogstashOutput"
    },
    "minemeld.ft.o365.O365XML": {
        "class": "minemeld.ft.o365:O365XML"
    },
    "minemeld.ft.o365.O365API": {
        "class": "minemeld.ft.o365:O365API"
    },
    "minemeld.ft.op.AggregateFT": {
        "class": "minemeld.ft.op:AggregateFT"
    },
    "minemeld.ft.phishme.Intelligence": {
        "class": "minemeld.ft.phishme:Intelligence"
    },
    "minemeld.ft.proofpoint.ETIntelligence": {
        "class": "minemeld.ft.proofpoint:ETIntelligence"
    },
    "minemeld.ft.proofpoint.EmergingThreatsIP": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsIP"
    },
    "minemeld.ft.proofpoint.EmergingThreatsDomain": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsDomain"
    },
    "minemeld.ft.recordedfuture.IPRiskList": {
        "class": "minemeld.ft.recordedfuture:IPRiskList"
    },
	"minemeld.ft.recordedfuture.DomainRiskList": {
        "class": "minemeld.ft.recordedfuture:DomainRiskList"
    },
    "minemeld.ft.redis.RedisSet": {
        "class": "minemeld.ft.redis:RedisSet"
    },
    "minemeld.ft.syslog.SyslogMatcher": {
        "class": "minemeld.ft.syslog:SyslogMatcher"
    },
    "minemeld.ft.syslog.SyslogMiner": {
        "class": "minemeld.ft.syslog:SyslogMiner"
    },
    "minemeld.ft.taxii.TaxiiClient": {
        "class": "minemeld.ft.taxii:TaxiiClient"
    },
    "minemeld.ft.taxii.DataFeed": {
        "class": "minemeld.ft.taxii:DataFeed"
    },
    "minemeld.ft.threatq.Export": {
        "class": "minemeld.ft.threatq:Export"
    },
    "minemeld.ft.tmt.DTIAPI": {
        "class": "minemeld.ft.tmt:DTIAPI"
    },
    "minemeld.ft.vt.Notifications": {
        "class": "minemeld.ft.vt:Notifications"
    },
    "minemeld.ft.mm.JSONSEQMiner": {
        "class": "minemeld.ft.mm:JSONSEQMiner"
    },
    "minemeld.ft.localdb.Miner": {
        "class": "minemeld.ft.localdb:Miner"
    },
    "minemeld.ft.threatconnect.IndicatorsMiner": {
        "class": "minemeld.ft.threatconnect:IndicatorsMiner"
    },
    "minemeld.ft.threatconnect.GroupsMiner": {
        "class": "minemeld.ft.threatconnect:GroupsMiner"
    },
    "minemeld.ft.visa.VTI": {
        "class": "minemeld.ft.visa:VTI"
    }
}

This is the .yml file I placed in the prototypes fodler:

 

{
    "minemeld.ft.anomali.Intelligence": {
        "class": "minemeld.ft.anomali:Intelligence"
    },
    "minemeld.ft.nslookupm.NSLOOKUPM":{
        "class": "minemeld.ft.nslookupm:NSLOOKUPM"
    }
    "minemeld.ft.auscert.MaliciousURLFeed": {
        "class": "minemeld.ft.auscert:MaliciousURLFeed"
    },
    "minemeld.ft.autofocus.ExportList": {
        "class": "minemeld.ft.autofocus:ExportList"
    },
    "minemeld.ft.azure.AzureXML": {
        "class": "minemeld.ft.azure:AzureXML"
    },
    "minemeld.ft.cif.Feed": {
        "class": "minemeld.ft.cif:Feed"
    },
    "minemeld.ft.ciscoise.ErsSgt": {
        "class": "minemeld.ft.ciscoise:ErsSgt"
    },
    "minemeld.ft.csv.CSVFT": {
        "class": "minemeld.ft.csv:CSVFT"
    },
    "minemeld.ft.dag.DagPusher": {
        "class": "minemeld.ft.dag:DagPusher"
    },
    "minemeld.ft.google.GoogleNetBlocks": {
        "class": "minemeld.ft.google:GoogleNetBlocks"
    },
    "minemeld.ft.google.GoogleCloudNetBlocks": {
        "class": "minemeld.ft.google:GoogleCloudNetBlocks"
    },
    "minemeld.ft.google.GoogleSPF": {
        "class": "minemeld.ft.google:GoogleSPF"
    },
    "minemeld.ft.http.HttpFT": {
        "class": "minemeld.ft.http:HttpFT"
    },
    "minemeld.ft.ipop.AggregateIPv4FT": {
        "class": "minemeld.ft.ipop:AggregateIPv4FT"
    },
    "minemeld.ft.json.SimpleJSON": {
        "class": "minemeld.ft.json:SimpleJSON"
    },
    "minemeld.ft.local.YamlFT": {
        "class": "minemeld.ft.local:YamlFT"
    },
    "minemeld.ft.local.YamlIPv4FT": {
        "class": "minemeld.ft.local:YamlIPv4FT"
    },
    "minemeld.ft.local.YamlURLFT": {
        "class": "minemeld.ft.local:YamlURLFT"
    },
    "minemeld.ft.local.YamlDomainFT": {
        "class": "minemeld.ft.local:YamlDomainFT"
    },
    "minemeld.ft.local.YamlIPv6FT": {
        "class": "minemeld.ft.local:YamlIPv6FT"
    },
    "minemeld.ft.logstash.LogstashOutput": {
        "class": "minemeld.ft.logstash:LogstashOutput"
    },
    "minemeld.ft.o365.O365XML": {
        "class": "minemeld.ft.o365:O365XML"
    },
    "minemeld.ft.o365.O365API": {
        "class": "minemeld.ft.o365:O365API"
    },
    "minemeld.ft.op.AggregateFT": {
        "class": "minemeld.ft.op:AggregateFT"
    },
    "minemeld.ft.phishme.Intelligence": {
        "class": "minemeld.ft.phishme:Intelligence"
    },
    "minemeld.ft.proofpoint.ETIntelligence": {
        "class": "minemeld.ft.proofpoint:ETIntelligence"
    },
    "minemeld.ft.proofpoint.EmergingThreatsIP": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsIP"
    },
    "minemeld.ft.proofpoint.EmergingThreatsDomain": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsDomain"
    },
    "minemeld.ft.recordedfuture.IPRiskList": {
        "class": "minemeld.ft.recordedfuture:IPRiskList"
    },
	"minemeld.ft.recordedfuture.DomainRiskList": {
        "class": "minemeld.ft.recordedfuture:DomainRiskList"
    },
    "minemeld.ft.redis.RedisSet": {
        "class": "minemeld.ft.redis:RedisSet"
    },
    "minemeld.ft.syslog.SyslogMatcher": {
        "class": "minemeld.ft.syslog:SyslogMatcher"
    },
    "minemeld.ft.syslog.SyslogMiner": {
        "class": "minemeld.ft.syslog:SyslogMiner"
    },
    "minemeld.ft.taxii.TaxiiClient": {
        "class": "minemeld.ft.taxii:TaxiiClient"
    },
    "minemeld.ft.taxii.DataFeed": {
        "class": "minemeld.ft.taxii:DataFeed"
    },
    "minemeld.ft.threatq.Export": {
        "class": "minemeld.ft.threatq:Export"
    },
    "minemeld.ft.tmt.DTIAPI": {
        "class": "minemeld.ft.tmt:DTIAPI"
    },
    "minemeld.ft.vt.Notifications": {
        "class": "minemeld.ft.vt:Notifications"
    },
    "minemeld.ft.mm.JSONSEQMiner": {
        "class": "minemeld.ft.mm:JSONSEQMiner"
    },
    "minemeld.ft.localdb.Miner": {
        "class": "minemeld.ft.localdb:Miner"
    },
    "minemeld.ft.threatconnect.IndicatorsMiner": {
        "class": "minemeld.ft.threatconnect:IndicatorsMiner"
    },
    "minemeld.ft.threatconnect.GroupsMiner": {
        "class": "minemeld.ft.threatconnect:GroupsMiner"
    },
    "minemeld.ft.visa.VTI": {
        "class": "minemeld.ft.visa:VTI"
    }
}

This is the minemeldlocal.yml file:

{
    "minemeld.ft.anomali.Intelligence": {
        "class": "minemeld.ft.anomali:Intelligence"
    },
    "minemeld.ft.nslookupm.NSLOOKUPM":{
        "class": "minemeld.ft.nslookupm:NSLOOKUPM"
    }
    "minemeld.ft.auscert.MaliciousURLFeed": {
        "class": "minemeld.ft.auscert:MaliciousURLFeed"
    },
    "minemeld.ft.autofocus.ExportList": {
        "class": "minemeld.ft.autofocus:ExportList"
    },
    "minemeld.ft.azure.AzureXML": {
        "class": "minemeld.ft.azure:AzureXML"
    },
    "minemeld.ft.cif.Feed": {
        "class": "minemeld.ft.cif:Feed"
    },
    "minemeld.ft.ciscoise.ErsSgt": {
        "class": "minemeld.ft.ciscoise:ErsSgt"
    },
    "minemeld.ft.csv.CSVFT": {
        "class": "minemeld.ft.csv:CSVFT"
    },
    "minemeld.ft.dag.DagPusher": {
        "class": "minemeld.ft.dag:DagPusher"
    },
    "minemeld.ft.google.GoogleNetBlocks": {
        "class": "minemeld.ft.google:GoogleNetBlocks"
    },
    "minemeld.ft.google.GoogleCloudNetBlocks": {
        "class": "minemeld.ft.google:GoogleCloudNetBlocks"
    },
    "minemeld.ft.google.GoogleSPF": {
        "class": "minemeld.ft.google:GoogleSPF"
    },
    "minemeld.ft.http.HttpFT": {
        "class": "minemeld.ft.http:HttpFT"
    },
    "minemeld.ft.ipop.AggregateIPv4FT": {
        "class": "minemeld.ft.ipop:AggregateIPv4FT"
    },
    "minemeld.ft.json.SimpleJSON": {
        "class": "minemeld.ft.json:SimpleJSON"
    },
    "minemeld.ft.local.YamlFT": {
        "class": "minemeld.ft.local:YamlFT"
    },
    "minemeld.ft.local.YamlIPv4FT": {
        "class": "minemeld.ft.local:YamlIPv4FT"
    },
    "minemeld.ft.local.YamlURLFT": {
        "class": "minemeld.ft.local:YamlURLFT"
    },
    "minemeld.ft.local.YamlDomainFT": {
        "class": "minemeld.ft.local:YamlDomainFT"
    },
    "minemeld.ft.local.YamlIPv6FT": {
        "class": "minemeld.ft.local:YamlIPv6FT"
    },
    "minemeld.ft.logstash.LogstashOutput": {
        "class": "minemeld.ft.logstash:LogstashOutput"
    },
    "minemeld.ft.o365.O365XML": {
        "class": "minemeld.ft.o365:O365XML"
    },
    "minemeld.ft.o365.O365API": {
        "class": "minemeld.ft.o365:O365API"
    },
    "minemeld.ft.op.AggregateFT": {
        "class": "minemeld.ft.op:AggregateFT"
    },
    "minemeld.ft.phishme.Intelligence": {
        "class": "minemeld.ft.phishme:Intelligence"
    },
    "minemeld.ft.proofpoint.ETIntelligence": {
        "class": "minemeld.ft.proofpoint:ETIntelligence"
    },
    "minemeld.ft.proofpoint.EmergingThreatsIP": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsIP"
    },
    "minemeld.ft.proofpoint.EmergingThreatsDomain": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsDomain"
    },
    "minemeld.ft.recordedfuture.IPRiskList": {
        "class": "minemeld.ft.recordedfuture:IPRiskList"
    },
	"minemeld.ft.recordedfuture.DomainRiskList": {
        "class": "minemeld.ft.recordedfuture:DomainRiskList"
    },
    "minemeld.ft.redis.RedisSet": {
        "class": "minemeld.ft.redis:RedisSet"
    },
    "minemeld.ft.syslog.SyslogMatcher": {
        "class": "minemeld.ft.syslog:SyslogMatcher"
    },
    "minemeld.ft.syslog.SyslogMiner": {
        "class": "minemeld.ft.syslog:SyslogMiner"
    },
    "minemeld.ft.taxii.TaxiiClient": {
        "class": "minemeld.ft.taxii:TaxiiClient"
    },
    "minemeld.ft.taxii.DataFeed": {
        "class": "minemeld.ft.taxii:DataFeed"
    },
    "minemeld.ft.threatq.Export": {
        "class": "minemeld.ft.threatq:Export"
    },
    "minemeld.ft.tmt.DTIAPI": {
        "class": "minemeld.ft.tmt:DTIAPI"
    },
    "minemeld.ft.vt.Notifications": {
        "class": "minemeld.ft.vt:Notifications"
    },
    "minemeld.ft.mm.JSONSEQMiner": {
        "class": "minemeld.ft.mm:JSONSEQMiner"
    },
    "minemeld.ft.localdb.Miner": {
        "class": "minemeld.ft.localdb:Miner"
    },
    "minemeld.ft.threatconnect.IndicatorsMiner": {
        "class": "minemeld.ft.threatconnect:IndicatorsMiner"
    },
    "minemeld.ft.threatconnect.GroupsMiner": {
        "class": "minemeld.ft.threatconnect:GroupsMiner"
    },
    "minemeld.ft.visa.VTI": {
        "class": "minemeld.ft.visa:VTI"
    }
}

 

L0 Member

Re: Issues registering custom prototype and custom miner

The error I am getting specifically is that I am trying to commit the nslookupm prototype but I am getting the following error:

 

COMMIT FAILED: Unknown node class minemeld.ft.nslookupm.NSLOOKUPM in node-1553265796327

Like what you see?

Show your appreciation!

Click Like if a post is helpful to you or if you just want to show your support.

Click Accept as Solution to acknowledge that the answer to your question has been provided.

The button appears next to the replies on topics you’ve started. The member who gave the solution and all future visitors to this topic will appreciate it!

These simple actions take just seconds of your time, but go a long way in showing appreciation for community members and the Live Community as a whole!

The Live Community thanks you for your participation!