• Iguana 6
  • Previous Versions
  • API
  • Sample Code
  • Training
  • Create a Ticket
iNTERFACEWARE Help Center
  • Iguana 6
  • Previous Versions
  • API
  • Sample Code
  • Training
  • Create a Ticket

Code Repository

Home›Code Repository›dup.lua
Modules

dup.lua

Verified Featured
Added by iNTERFACEWARE

Duplicate message filter.

Source Code
-- Duplicate module.  When a channel first starts or you edit this script this module will query the history of
-- messages off the channel log.  After that it maintains a linked list of MD5 hashes of messages in memory.  The algorithms
-- are intended to be efficient - lookups done using a hashmap and maintaining the list of messages to expire using a linked list.
-- Since the whole thing operates in memory after startup it should be fast and the use of MD5 hashes rather than storing the raw
-- messages themselves limits the amount of memory overhead.
-- Read http://help.interfaceware.com/kb/duplicate-message-filter for more information.

local dup = {}

dup.UserName='admin'
dup.Password='password'

local Queue = {}
Queue.__index = Queue

function Queue.create()
  local q = {}
  setmetatable(q,Queue)
  q.size = 0
  q.head = nil
  q.tail = nil
  return q
end

-- Add node at end of queue
function Queue:enqueue(node)
  if self.size > 0 then
      self.tail.next = node
  else
      self.head      = node
  end
  self.tail = node
  self.size = self.size + 1
end

-- Remove node from front of queue and return it
function Queue:dequeue()
   if self.size > 0 then
      local removed = self.head
      self.head     = self.head.next
      self.size     = self.size - 1
      if self.size == 0 then self.tail = nil end
      return removed
   end
end

-- Retrieve node from front of queue without removing 
function Queue:peek()
   if self.size > 0 then return self.head end
end

-- Create a new Log Message node
function createMsgNode(Hash)
   return {hash=Hash, next=nil}
end

local function storeMessages(X)
   local LoggedMessage, LoggedMessageHash, LoggedMessageStamp = nil, nil, nil
   local numMessages = X.export:childCount('message')
   for i = numMessages, 1, -1  do
      LoggedMessage      = X.export:child("message", i).data:nodeValue()
      LoggedMessageHash  = util.md5(LoggedMessage)
      LoggedMessageStamp = X.export:child("message", i).time_stamp:nodeValue()
      
      --Store Message data in lookup and queue
      dup.lookup[LoggedMessageHash] = LoggedMessageStamp
      dup.queue:enqueue(createMsgNode(LoggedMessageHash)) 
   end    
   trace(dup.queue)   
end

local function fetchMessages(LookbackAmount)
   local protocol = iguana.webInfo().web_config.use_https and 'https' or 'http'
   local port     = iguana.webInfo().web_config.port
   trace(protocol) trace(port)
  
   local X = net.http.get{
      url    =  protocol .. '://localhost:' .. port .. '/api_query',         
      parameters = {
         username = dup.UserName, 
         password = dup.Password, 
         source = iguana.channelName(), 
         limit = LookbackAmount,
         type= 'messages',
         deleted = false,
         unique = 'true',
         reverse = 'true' 
      },
      live = true
   }
   X = xml.parse{data=X}
   return X
end

local function refreshStoredMessages(LookbackAmount)
   -- Remove oldest message so queue size equals lookback amount.
   if(dup.queue.size > LookbackAmount) then
      local removedMsg = dup.queue:dequeue()
      dup.lookup[removedMsg.hash] = nil
   end
end
   
local function getCurrentTimestamp()
   local  CurrentTime      = os.ts.gmtime()  
   local  CurrentTimestamp = os.ts.date("%Y/%m/%d %X", CurrentTime)
   return CurrentTimestamp
end

dup.lookup = {}
dup.queue = Queue.create()

function dup.isDuplicate(Params)
   -- Uncomment line below when testing annotations. 
   -- Will force queue to always be empty so fetch/store annotations
   -- can be viewed. 
   -- dup.queue = Queue.create()
   local Hash           = util.md5(Params.data)
   local LookbackAmount = Params.lookback_amount
   -- Populate in memory structures with lookbackAmount # of messages. 
   -- Messages are retrived from API in a table sorted newest -> oldest.
   if(dup.queue.size == 0) then
      iguana.logInfo('dup.lua: Fetching Initial Messages!')
      local messages = fetchMessages(LookbackAmount)
      iguana.logInfo('dup.lua: Storing Initial Messages!')
      storeMessages(messages)
      iguana.logInfo('dup.lua: Finished Initial fetch/store of '
         .. dup.queue.size .. ' messages!')
   else  
      refreshStoredMessages(LookbackAmount)
   end
   --Check if current message is in lookup table
   if dup.lookup[Hash] then
      return true
   else
      local timeStamp = getCurrentTimestamp()
      dup.lookup[Hash] = timeStamp
      dup.queue:enqueue(createMsgNode(Hash))
      trace(dup.queue.size)
      return false 
   end   
end

local HELP_DEF=[[{
   "Desc": "Checks to see if a message is a duplicate based off in memory buffer of MD5 hashes it gets from the log",
   "Returns": [{
         "Desc": "True if the message is detected as a duplicate"
      }
   ],
   "SummaryLine": "Is message a duplicate?",
   "SeeAlso": [
      {
         "Title": "Duplicate Filter.",
         "Link": "http://help.interfaceware.com/v6/duplicate-filter"
      },
      {
         "Title": "dup.lua - in our code repository.",
         "Link": "http://help.interfaceware.com/code/details/dup-lua"
      }
   ],
   "Title": "dup.isDuplicate",
   "Usage": "dup.isDuplicate{data=<value>, lookback_amount=<number>}",
   "Parameters": [
      {
         "data": {
            "Desc": "A message to be checked to see if it is a duplicate <u>string</u>. "
         }
      },
      {
         "lookback_amount": {
            "Desc": "Number of messages to look back at to check for duplicates <u>number</u>. "
         }
      }
   ],
   "Examples": [
      "<pre>local IsDuplicate = dup.isDuplicate{data=Data, lookback_amount=800}</pre>"
   ],
   "ParameterTable": true
}]]

help.set{input_function=dup.isDuplicate, help_data=json.parse{data=HELP_DEF}}    
  

return dup
Description
Duplicate message filter.
Attachments
dup_Filter.zip
Usage Details

The dup.lua module is an efficient filter to eliminate duplicate messages. When the script starts up it queries the Iguana log for the last N messages that you specify. From then on it maintains a list of MD5 hashes for the last N messages in a first in, first out (FIFO) order. A message is a duplicate if it’s MD5 hash matches the MD5 hash of a previous message.

Load the project into a Filter component, and click through the data to process sample messages. Attempting to process a message twice will find a duplicate. To process a message twice press the execute button again, or step back to a previously processed message.

How to use dup.lua:

  • Add it to your shared modules in any Translator project
  • Make sure to add dup = require 'dup' at the top of your script
  • Use the dup.isDuplicate{data=Data, lookback_amount=###} function to compare the incoming message with previous ones

Sample code for main():

dup = require 'dup'
-- This shows the usage of the duplicate filter
-- https://help.interfaceware.com/code/details/dup-lua

function main(Data)
   iguana.logInfo("Duplicate filter received message!")
   -- lookback_amount is the amount of previous messages to look back 
   -- through the logs for a duplicate.
   local Success = dup.isDuplicate{data=Data, lookback_amount=800}
  
   if Success then
      iguana.logInfo('Found a Duplicate!')
      return    
   end
   -- If it's not a duplicate then push the message into the queue
   queue.push{data=Data}
end
More Information
Filter out duplicate messages
Bookmark
  • Reviews
  • Related Listings
Filter
Sort by: Newest First
  • Oldest First
  • Rating
  • Helpfulness
Write a Review
Rating
Keyword
Filter
Sort by: Title
  • Newest First
  • Oldest First
  • Most Reviews
  • Highest Rated
Rating
iNTERFACEWARE
sqlite.lua
Added by iNTERFACEWARE
Modules
A module to create a SQLite database and generate the tables specified in a VMD
store.lua
Added by iNTERFACEWARE
Modules
The "original" store module: Allows you to store key/value pairs in a persistent storage mechanism. We recommend using the new store2 module instead.
store2.lua
Added by iNTERFACEWARE
Modules
Provides a simple interface to store key/value pairs in a persistent storage mechanism.
stream.lua
Added by iNTERFACEWARE
Modules
This module performs basic stream processing, read, write, save to file, convert to text etc.
stringutil.lua
Added by iNTERFACEWARE
Modules
A library of helpful extensions to the standard Lua string library.
test_all.lua
Added by iNTERFACEWARE
Modules
Test a script against all the sample messages loaded for the component
throttle.lua
Added by iNTERFACEWARE
Modules
Throttle a process during peak hours, by slowing down the code.
throttleDB.lua
Added by iNTERFACEWARE
Modules
Throttle database access by reducing the number of inserts during peak hours
urlcode.lua
Added by iNTERFACEWARE
Modules
A module for parsing URL encoded GET/POST sequences
validate.lua
Added by iNTERFACEWARE
Modules
A template module for testing HL7 message conformance, you will need to extend it to match your requirements
xml.lua
Added by iNTERFACEWARE
Modules
A collection of helpful XML node functions.
Showing 21 - 31 of 31 results
«12»

Topics

  • expandGetting Started
  • expandAdministration
    • expandInstallation
    • expandLicensing
    • expandUpgrades
    • expandDeployment
    • expandConfiguration Management
      • expandCustom Configuration
    • expandBackup and Restore
    • expandSecurity
      • expandHIPAA Compliance
    • expandTroubleshooting
  • expandDeveloping Interfaces
    • expandArchitecture
    • expandInterfaces
      • expandHL7
      • expandDatabase
        • expandConnect
      • expandWeb Services
      • expandCDA
      • expandX12
      • expandOther Interfaces
      • expandUtilities
    • expandRepositories
      • expandBuiltin Repositories
        • expandIguana Upgrade
        • expandIguana Tutorials
        • expandIguana Tools
        • expandIguana Protocols
        • expandIguana Files
        • expandIguana Date/Time
        • expandIguana Webservices
        • expandIguana Excel
      • expandRemote Repositories
      • expandCS Team Repositories
        • expandIguana Channels
    • expandSample Code
      • expandModules
      • expandUsing built-in functions
      • expandWorking with XML
    • expandLua Programming
    • expandPerformance
  • expandFAQs and TIPs
    • expandFrequently Asked Questions
      • expandInstalls and Upgrades
      • expandWeb Services
      • expandConfiguration
      • expandChannels
      • expandTranslator
      • expandOther
      • expandDatabase
      • expandAdministration
      • expandLogs
      • expandChameleon
    • expandTips
      • expandChannels
      • expandChameleon
      • expandWeb Services
      • expandSecurity
      • expandProgramming
      • expandOther
      • expandAdministration
  • expandReference
    • expandIguana Enterprise and Professional
    • expandProgram Settings
    • expandChannel Settings
    • expandDashboard
    • expandChannels
    • expandTranslator
    • expandLogs
      • expandLog Encryption
    • expandHTTP API
    • expandCDA API
    • expandError Messages
    • expandChameleon
    • expandIguana Change Log

Other Links

  • Training Center
  • News & Announcements
  • iNTERFACEWARE Blog
  • Older Documention (IGUANA v4 & Chameleon)
Copyright © iNTERFACEWARE Inc.