This website uses Cookies. By clicking Accept, you agree to the storing of cookies on your device to enhance your community and translation experience. Read our Privacy Policy.
Click Preferences to customize your cookie settings.
Accept
Reject
Preferences

Issues registering custom prototype and custom miner

cancel
Turn on suggestions
Auto-suggest helps you quickly narrow down your search results by suggesting possible matches as you type.
Showing results for 
Show  only  | Search instead for 
Did you mean: 
Announcements

Issues registering custom prototype and custom miner

account_admin
L0 Member

‎03-22-2019 10:52 AM 

  1. # /opt/minemeld/engine/core/minemeld/nslookupm.py

from __future__ import absolute_import

import logging
#import requests
import socket

#from . import basepoller

LOG = logging.getLogger(__name__)


class NSLOOKUPM():
    def configure(self):
        # super(nsLookupM, self).configure()

        # self.polling_timeout = self.config.get('polling_timeout', 20)
        # self.verify_cert = self.config.get('verify_cert', True)

        # self.channel_name = self.config.get('channel_name', None)
        # if self.channel_name is None:
        #     raise ValueError('%s - channel name is required' % self.name)
        # self.url = 'https://www.youtube.com/user/{}/videos'.format{
        #    self.channel_name
        # }
        self.domain = 'dhswcms.upload.akamai.com'

        #self._build_iterator

    def _build_iterator(self):
        # called at every polling interval
        # here you should retrieve and return the list of item
        ip_list = socket.gethostbyname_ex(self.domain)[2]
        print(ip_list)
        return ip_list 

    def _process_item(self):
        # called on each item returned by _build_iterator
        # it should return a list of (indicator, value) pairs
        indicator_list = []
        for ipaddr in self._build_iterator():
            value = {
                'type': 'IP',
                'confidence': 100
            }
            indicator_list.append([ipaddr, value])
        return indicator_list 

# For testing purposes
nsLookup = NSLOOKUPM() # must configure first
nsLookup.configure() # configure
print(nsLookup._process_item()) #  process step




#test = nsLookupM()
#test()
#test._process_item()
#test._build_iterator()
    I have gone through the steps to create a custom prototype and I have tried to make sure and followed the advice on these forums and the Youtube tutorial. Above is my customer prototype that I placed in the ft folder.

Below is the  nodes.json file in the core folder:

 

{
    "minemeld.ft.anomali.Intelligence": {
        "class": "minemeld.ft.anomali:Intelligence"
    },
    "minemeld.ft.nslookupm.NSLOOKUPM":{
        "class": "minemeld.ft.nslookupm:NSLOOKUPM"
    }
    "minemeld.ft.auscert.MaliciousURLFeed": {
        "class": "minemeld.ft.auscert:MaliciousURLFeed"
    },
    "minemeld.ft.autofocus.ExportList": {
        "class": "minemeld.ft.autofocus:ExportList"
    },
    "minemeld.ft.azure.AzureXML": {
        "class": "minemeld.ft.azure:AzureXML"
    },
    "minemeld.ft.cif.Feed": {
        "class": "minemeld.ft.cif:Feed"
    },
    "minemeld.ft.ciscoise.ErsSgt": {
        "class": "minemeld.ft.ciscoise:ErsSgt"
    },
    "minemeld.ft.csv.CSVFT": {
        "class": "minemeld.ft.csv:CSVFT"
    },
    "minemeld.ft.dag.DagPusher": {
        "class": "minemeld.ft.dag:DagPusher"
    },
    "minemeld.ft.google.GoogleNetBlocks": {
        "class": "minemeld.ft.google:GoogleNetBlocks"
    },
    "minemeld.ft.google.GoogleCloudNetBlocks": {
        "class": "minemeld.ft.google:GoogleCloudNetBlocks"
    },
    "minemeld.ft.google.GoogleSPF": {
        "class": "minemeld.ft.google:GoogleSPF"
    },
    "minemeld.ft.http.HttpFT": {
        "class": "minemeld.ft.http:HttpFT"
    },
    "minemeld.ft.ipop.AggregateIPv4FT": {
        "class": "minemeld.ft.ipop:AggregateIPv4FT"
    },
    "minemeld.ft.json.SimpleJSON": {
        "class": "minemeld.ft.json:SimpleJSON"
    },
    "minemeld.ft.local.YamlFT": {
        "class": "minemeld.ft.local:YamlFT"
    },
    "minemeld.ft.local.YamlIPv4FT": {
        "class": "minemeld.ft.local:YamlIPv4FT"
    },
    "minemeld.ft.local.YamlURLFT": {
        "class": "minemeld.ft.local:YamlURLFT"
    },
    "minemeld.ft.local.YamlDomainFT": {
        "class": "minemeld.ft.local:YamlDomainFT"
    },
    "minemeld.ft.local.YamlIPv6FT": {
        "class": "minemeld.ft.local:YamlIPv6FT"
    },
    "minemeld.ft.logstash.LogstashOutput": {
        "class": "minemeld.ft.logstash:LogstashOutput"
    },
    "minemeld.ft.o365.O365XML": {
        "class": "minemeld.ft.o365:O365XML"
    },
    "minemeld.ft.o365.O365API": {
        "class": "minemeld.ft.o365:O365API"
    },
    "minemeld.ft.op.AggregateFT": {
        "class": "minemeld.ft.op:AggregateFT"
    },
    "minemeld.ft.phishme.Intelligence": {
        "class": "minemeld.ft.phishme:Intelligence"
    },
    "minemeld.ft.proofpoint.ETIntelligence": {
        "class": "minemeld.ft.proofpoint:ETIntelligence"
    },
    "minemeld.ft.proofpoint.EmergingThreatsIP": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsIP"
    },
    "minemeld.ft.proofpoint.EmergingThreatsDomain": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsDomain"
    },
    "minemeld.ft.recordedfuture.IPRiskList": {
        "class": "minemeld.ft.recordedfuture:IPRiskList"
    },
	"minemeld.ft.recordedfuture.DomainRiskList": {
        "class": "minemeld.ft.recordedfuture:DomainRiskList"
    },
    "minemeld.ft.redis.RedisSet": {
        "class": "minemeld.ft.redis:RedisSet"
    },
    "minemeld.ft.syslog.SyslogMatcher": {
        "class": "minemeld.ft.syslog:SyslogMatcher"
    },
    "minemeld.ft.syslog.SyslogMiner": {
        "class": "minemeld.ft.syslog:SyslogMiner"
    },
    "minemeld.ft.taxii.TaxiiClient": {
        "class": "minemeld.ft.taxii:TaxiiClient"
    },
    "minemeld.ft.taxii.DataFeed": {
        "class": "minemeld.ft.taxii:DataFeed"
    },
    "minemeld.ft.threatq.Export": {
        "class": "minemeld.ft.threatq:Export"
    },
    "minemeld.ft.tmt.DTIAPI": {
        "class": "minemeld.ft.tmt:DTIAPI"
    },
    "minemeld.ft.vt.Notifications": {
        "class": "minemeld.ft.vt:Notifications"
    },
    "minemeld.ft.mm.JSONSEQMiner": {
        "class": "minemeld.ft.mm:JSONSEQMiner"
    },
    "minemeld.ft.localdb.Miner": {
        "class": "minemeld.ft.localdb:Miner"
    },
    "minemeld.ft.threatconnect.IndicatorsMiner": {
        "class": "minemeld.ft.threatconnect:IndicatorsMiner"
    },
    "minemeld.ft.threatconnect.GroupsMiner": {
        "class": "minemeld.ft.threatconnect:GroupsMiner"
    },
    "minemeld.ft.visa.VTI": {
        "class": "minemeld.ft.visa:VTI"
    }
}

This is the .yml file I placed in the prototypes fodler:

 

{
    "minemeld.ft.anomali.Intelligence": {
        "class": "minemeld.ft.anomali:Intelligence"
    },
    "minemeld.ft.nslookupm.NSLOOKUPM":{
        "class": "minemeld.ft.nslookupm:NSLOOKUPM"
    }
    "minemeld.ft.auscert.MaliciousURLFeed": {
        "class": "minemeld.ft.auscert:MaliciousURLFeed"
    },
    "minemeld.ft.autofocus.ExportList": {
        "class": "minemeld.ft.autofocus:ExportList"
    },
    "minemeld.ft.azure.AzureXML": {
        "class": "minemeld.ft.azure:AzureXML"
    },
    "minemeld.ft.cif.Feed": {
        "class": "minemeld.ft.cif:Feed"
    },
    "minemeld.ft.ciscoise.ErsSgt": {
        "class": "minemeld.ft.ciscoise:ErsSgt"
    },
    "minemeld.ft.csv.CSVFT": {
        "class": "minemeld.ft.csv:CSVFT"
    },
    "minemeld.ft.dag.DagPusher": {
        "class": "minemeld.ft.dag:DagPusher"
    },
    "minemeld.ft.google.GoogleNetBlocks": {
        "class": "minemeld.ft.google:GoogleNetBlocks"
    },
    "minemeld.ft.google.GoogleCloudNetBlocks": {
        "class": "minemeld.ft.google:GoogleCloudNetBlocks"
    },
    "minemeld.ft.google.GoogleSPF": {
        "class": "minemeld.ft.google:GoogleSPF"
    },
    "minemeld.ft.http.HttpFT": {
        "class": "minemeld.ft.http:HttpFT"
    },
    "minemeld.ft.ipop.AggregateIPv4FT": {
        "class": "minemeld.ft.ipop:AggregateIPv4FT"
    },
    "minemeld.ft.json.SimpleJSON": {
        "class": "minemeld.ft.json:SimpleJSON"
    },
    "minemeld.ft.local.YamlFT": {
        "class": "minemeld.ft.local:YamlFT"
    },
    "minemeld.ft.local.YamlIPv4FT": {
        "class": "minemeld.ft.local:YamlIPv4FT"
    },
    "minemeld.ft.local.YamlURLFT": {
        "class": "minemeld.ft.local:YamlURLFT"
    },
    "minemeld.ft.local.YamlDomainFT": {
        "class": "minemeld.ft.local:YamlDomainFT"
    },
    "minemeld.ft.local.YamlIPv6FT": {
        "class": "minemeld.ft.local:YamlIPv6FT"
    },
    "minemeld.ft.logstash.LogstashOutput": {
        "class": "minemeld.ft.logstash:LogstashOutput"
    },
    "minemeld.ft.o365.O365XML": {
        "class": "minemeld.ft.o365:O365XML"
    },
    "minemeld.ft.o365.O365API": {
        "class": "minemeld.ft.o365:O365API"
    },
    "minemeld.ft.op.AggregateFT": {
        "class": "minemeld.ft.op:AggregateFT"
    },
    "minemeld.ft.phishme.Intelligence": {
        "class": "minemeld.ft.phishme:Intelligence"
    },
    "minemeld.ft.proofpoint.ETIntelligence": {
        "class": "minemeld.ft.proofpoint:ETIntelligence"
    },
    "minemeld.ft.proofpoint.EmergingThreatsIP": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsIP"
    },
    "minemeld.ft.proofpoint.EmergingThreatsDomain": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsDomain"
    },
    "minemeld.ft.recordedfuture.IPRiskList": {
        "class": "minemeld.ft.recordedfuture:IPRiskList"
    },
	"minemeld.ft.recordedfuture.DomainRiskList": {
        "class": "minemeld.ft.recordedfuture:DomainRiskList"
    },
    "minemeld.ft.redis.RedisSet": {
        "class": "minemeld.ft.redis:RedisSet"
    },
    "minemeld.ft.syslog.SyslogMatcher": {
        "class": "minemeld.ft.syslog:SyslogMatcher"
    },
    "minemeld.ft.syslog.SyslogMiner": {
        "class": "minemeld.ft.syslog:SyslogMiner"
    },
    "minemeld.ft.taxii.TaxiiClient": {
        "class": "minemeld.ft.taxii:TaxiiClient"
    },
    "minemeld.ft.taxii.DataFeed": {
        "class": "minemeld.ft.taxii:DataFeed"
    },
    "minemeld.ft.threatq.Export": {
        "class": "minemeld.ft.threatq:Export"
    },
    "minemeld.ft.tmt.DTIAPI": {
        "class": "minemeld.ft.tmt:DTIAPI"
    },
    "minemeld.ft.vt.Notifications": {
        "class": "minemeld.ft.vt:Notifications"
    },
    "minemeld.ft.mm.JSONSEQMiner": {
        "class": "minemeld.ft.mm:JSONSEQMiner"
    },
    "minemeld.ft.localdb.Miner": {
        "class": "minemeld.ft.localdb:Miner"
    },
    "minemeld.ft.threatconnect.IndicatorsMiner": {
        "class": "minemeld.ft.threatconnect:IndicatorsMiner"
    },
    "minemeld.ft.threatconnect.GroupsMiner": {
        "class": "minemeld.ft.threatconnect:GroupsMiner"
    },
    "minemeld.ft.visa.VTI": {
        "class": "minemeld.ft.visa:VTI"
    }
}

This is the minemeldlocal.yml file:

{
    "minemeld.ft.anomali.Intelligence": {
        "class": "minemeld.ft.anomali:Intelligence"
    },
    "minemeld.ft.nslookupm.NSLOOKUPM":{
        "class": "minemeld.ft.nslookupm:NSLOOKUPM"
    }
    "minemeld.ft.auscert.MaliciousURLFeed": {
        "class": "minemeld.ft.auscert:MaliciousURLFeed"
    },
    "minemeld.ft.autofocus.ExportList": {
        "class": "minemeld.ft.autofocus:ExportList"
    },
    "minemeld.ft.azure.AzureXML": {
        "class": "minemeld.ft.azure:AzureXML"
    },
    "minemeld.ft.cif.Feed": {
        "class": "minemeld.ft.cif:Feed"
    },
    "minemeld.ft.ciscoise.ErsSgt": {
        "class": "minemeld.ft.ciscoise:ErsSgt"
    },
    "minemeld.ft.csv.CSVFT": {
        "class": "minemeld.ft.csv:CSVFT"
    },
    "minemeld.ft.dag.DagPusher": {
        "class": "minemeld.ft.dag:DagPusher"
    },
    "minemeld.ft.google.GoogleNetBlocks": {
        "class": "minemeld.ft.google:GoogleNetBlocks"
    },
    "minemeld.ft.google.GoogleCloudNetBlocks": {
        "class": "minemeld.ft.google:GoogleCloudNetBlocks"
    },
    "minemeld.ft.google.GoogleSPF": {
        "class": "minemeld.ft.google:GoogleSPF"
    },
    "minemeld.ft.http.HttpFT": {
        "class": "minemeld.ft.http:HttpFT"
    },
    "minemeld.ft.ipop.AggregateIPv4FT": {
        "class": "minemeld.ft.ipop:AggregateIPv4FT"
    },
    "minemeld.ft.json.SimpleJSON": {
        "class": "minemeld.ft.json:SimpleJSON"
    },
    "minemeld.ft.local.YamlFT": {
        "class": "minemeld.ft.local:YamlFT"
    },
    "minemeld.ft.local.YamlIPv4FT": {
        "class": "minemeld.ft.local:YamlIPv4FT"
    },
    "minemeld.ft.local.YamlURLFT": {
        "class": "minemeld.ft.local:YamlURLFT"
    },
    "minemeld.ft.local.YamlDomainFT": {
        "class": "minemeld.ft.local:YamlDomainFT"
    },
    "minemeld.ft.local.YamlIPv6FT": {
        "class": "minemeld.ft.local:YamlIPv6FT"
    },
    "minemeld.ft.logstash.LogstashOutput": {
        "class": "minemeld.ft.logstash:LogstashOutput"
    },
    "minemeld.ft.o365.O365XML": {
        "class": "minemeld.ft.o365:O365XML"
    },
    "minemeld.ft.o365.O365API": {
        "class": "minemeld.ft.o365:O365API"
    },
    "minemeld.ft.op.AggregateFT": {
        "class": "minemeld.ft.op:AggregateFT"
    },
    "minemeld.ft.phishme.Intelligence": {
        "class": "minemeld.ft.phishme:Intelligence"
    },
    "minemeld.ft.proofpoint.ETIntelligence": {
        "class": "minemeld.ft.proofpoint:ETIntelligence"
    },
    "minemeld.ft.proofpoint.EmergingThreatsIP": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsIP"
    },
    "minemeld.ft.proofpoint.EmergingThreatsDomain": {
        "class": "minemeld.ft.proofpoint:EmergingThreatsDomain"
    },
    "minemeld.ft.recordedfuture.IPRiskList": {
        "class": "minemeld.ft.recordedfuture:IPRiskList"
    },
	"minemeld.ft.recordedfuture.DomainRiskList": {
        "class": "minemeld.ft.recordedfuture:DomainRiskList"
    },
    "minemeld.ft.redis.RedisSet": {
        "class": "minemeld.ft.redis:RedisSet"
    },
    "minemeld.ft.syslog.SyslogMatcher": {
        "class": "minemeld.ft.syslog:SyslogMatcher"
    },
    "minemeld.ft.syslog.SyslogMiner": {
        "class": "minemeld.ft.syslog:SyslogMiner"
    },
    "minemeld.ft.taxii.TaxiiClient": {
        "class": "minemeld.ft.taxii:TaxiiClient"
    },
    "minemeld.ft.taxii.DataFeed": {
        "class": "minemeld.ft.taxii:DataFeed"
    },
    "minemeld.ft.threatq.Export": {
        "class": "minemeld.ft.threatq:Export"
    },
    "minemeld.ft.tmt.DTIAPI": {
        "class": "minemeld.ft.tmt:DTIAPI"
    },
    "minemeld.ft.vt.Notifications": {
        "class": "minemeld.ft.vt:Notifications"
    },
    "minemeld.ft.mm.JSONSEQMiner": {
        "class": "minemeld.ft.mm:JSONSEQMiner"
    },
    "minemeld.ft.localdb.Miner": {
        "class": "minemeld.ft.localdb:Miner"
    },
    "minemeld.ft.threatconnect.IndicatorsMiner": {
        "class": "minemeld.ft.threatconnect:IndicatorsMiner"
    },
    "minemeld.ft.threatconnect.GroupsMiner": {
        "class": "minemeld.ft.threatconnect:GroupsMiner"
    },
    "minemeld.ft.visa.VTI": {
        "class": "minemeld.ft.visa:VTI"
    }
}

 

0 Likes Likes
1 REPLY 1

account_admin
L0 Member

‎03-22-2019 10:53 AM

The error I am getting specifically is that I am trying to commit the nslookupm prototype but I am getting the following error:

 

COMMIT FAILED: Unknown node class minemeld.ft.nslookupm.NSLOOKUPM in node-1553265796327

0 Likes Likes
  • 2591 Views
  • 1 replies
  • 0 Likes
Like what you see?

Show your appreciation!

Click Like if a post is helpful to you or if you just want to show your support.

Click Accept as Solution to acknowledge that the answer to your question has been provided.

The button appears next to the replies on topics you’ve started. The member who gave the solution and all future visitors to this topic will appreciate it!

These simple actions take just seconds of your time, but go a long way in showing appreciation for community members and the LIVEcommunity as a whole!

The LIVEcommunity thanks you for your participation!

LEGAL NOTICES
RESOURCES
Khoros awards 2022
Copyright 2007 - 2024 - Palo Alto Networks