Slow response on CustomSelector results in unique ID error

Enonic version: 7.2.4 & 7.4.0
OS: Ubuntu

Hi,

I have a CustomSelector service that have to do multiple HTTP GET requests and therefore takes a while to respond.
From what I’ve seen so far while working in this, previous results (including those when I’m still typing a query text) seem to be considered at the final list.
I mean, if I’m typing “gravi”, it will make requests for “gra”, “grav” and “gravi”, and, although each request generates unique IDs on its own, I get the unique ID error. Some results are common to all three requests (“graviditet”, for instance).

Right after receiving this error, if I just click the expand arrow button it usually shows me the last terms correctly and no error, but I can’t rely the customer will understand that.
Is there something that can be done?

Regards,
Marco

Can you share the code of your custom selector?

Hi,

Sure thing.
I’ve merged some functions from the libs so it should be easier to understand what we’re doing.

It needs to fetch some values from Snowstorm API, but, since this first request doesn’t provide the preferred Norwegian terms, we need to make a second request using the ID from the first one. It means that the complexity is around O(n^2), so it takes a while to finish.
I’ve included a stored objects solution to speed-up the selector response, but new terms still take too long to return (when the objects still don’t exist).

var objectsLib = require('/lib/util/objects')
const utilLib = require('/lib/util/util')

const repo = utilLib.connectDatabase()

exports.get = handleGet

function handleGet (req) {
  const params = parseParams(req.params)
  const body = createResults(getSnomedCode(params), params, fetchSnomedCodes)

  return {
    contentType: 'application/json',
    body: body
  }
}

function createResults (items, params, fetchFn) {
  if (params && !params.query && params.ids) {
    return fetchFn(params.ids)
  } else {
    return items
  }
}

function parseParams (params) {
  var query = params.query
  var ids
  var start
  var count

  try {
    ids = (params.ids ? params.ids.split(',') : [])
  } catch (e) {
    utilLib.log('Invalid parameter ids: %s, using []', params.ids, 'warning')
    ids = []
  }

  try {
    start = Math.max(parseInt(params.start) || 0, 0)
  } catch (e) {
    utilLib.log('Invalid parameter start: %s, using 0', params.start, 'warning')
    start = 0
  }

  try {
    count = Math.max(parseInt(params.count) || 15, 0)
  } catch (e) {
    utilLib.log('Invalid parameter count: %s, using 15', params.count, 'warning')
    count = 15
  }

  return {
    query: query,
    ids: ids,
    start: start,
    end: start + count,
    count: count
  }
}


function getSnomedCode (params) {
  const text = params.query
  const hits = []

  if (text) {
    const response = queryInSnomedAPI(text)
    // if it's not an array, create one and include this object
    const items = objectsLib.forceArray(response && response.items)

    items.forEach(item => {
      if (!hits.some(el => el.id === item.concept.conceptId)) {
        // speeding-up the selector using stored objects
        const conceptQuery = repo.query({
          query: `data.conceptId='${item.concept.conceptId}'`,
          count: 1
        })

        if (conceptQuery.total > 0) {
          // speeding-up the selector using stored objects
          const conceptObj = repo.get({ key: conceptQuery.hits[0].id })
          hits.push({
            id: conceptObj.data.conceptId,
            displayName: conceptObj.data && conceptObj.data.displayNameSelector,
            description: conceptObj.data && conceptObj.data.descriptionSelector
          })
        } else {
          const conceptResponse = getConceptInAPI(item.concept.conceptId)
          if (!conceptResponse.error) {
            const descriptions = objectsLib.forceArray(conceptResponse && conceptResponse.descriptions).filter(el => el.conceptId === conceptResponse.conceptId)

            const hitObj = {
              id: conceptResponse.conceptId,
              description: `FSN Term: ${conceptResponse.fsn.term}; SCTID: ${conceptResponse.conceptId}`
            }

            // looks for PREFERRED
            let acceptabilityList = descriptions.filter(el => (el.lang === 'nb' || el.lang === 'no' || el.lang === 'nn') && objectsLib.valuesToArray(el.acceptabilityMap).some(it => it === 'PREFERRED'))
            if (acceptabilityList.length > 0) {
              hitObj.displayName = acceptabilityList[0].term
            } else {
            // looks for ACCEPTABLE if there's no PREFERRED
              acceptabilityList = descriptions.filter(el => (el.lang === 'nb' || el.lang === 'no' || el.lang === 'no') && objectsLib.valuesToArray(el.acceptabilityMap).some(it => it === 'ACCEPTABLE'))
              if (acceptabilityList.length > 0) {
                hitObj.displayName = acceptabilityList[0].term
              } else {
              // just use whatever is available
                hitObj.displayName = item.term
              }
            }

            hits.push(hitObj)
          }
        }
      }
    })

    return {
      hits: hits,
      total: hits.length,
      count: hits.length
    }
  } else {
    return {
      hits: [],
      total: 0,
      count: 0
    }
  }
}

function fetchSnomedCodes (ids) {
  const hits = []

  ids.forEach(id => {
    const conceptQuery = repo.query({
      query: `data.conceptId='${id}'`,
      count: 1
    })

    if (conceptQuery.total > 0) {
      const conceptObj = repo.get(conceptQuery.hits[0].id)
      hits.push({
        id: id,
        displayName: (conceptObj.data && conceptObj.data.displayNameSelector) || (conceptObj.data && conceptObj.data.korttittel) || id,
        description: (conceptObj.data && conceptObj.data.descriptionSelector) || (conceptObj.data && conceptObj.data.tittel) || 'Not imported yet'
      })
    } else {
      hits.push({
        id: id,
        displayName: id,
        description: 'Not imported yet'
      })
    }
  })

  return {
    count: ids.length,
    total: ids.length,
    hits: hits
  }
}

function queryInSnomedAPI (text) {
  // Request by search word
  const url = `https://snowstorm.rundberg.no/browser/MAIN/SNOMEDCT-NO-EXTENDED/descriptions?limit=8&active=true&groupByConcept=true&language=no&language=nb&language=nn&conceptActive=true&conceptRefset=&term=${text}`
  return sendQuerySnomedAPI(url)
}

function getConceptInAPI (conceptId) {
  // Request Concept
  const url = `https://snowstorm.rundberg.no/browser/MAIN/SNOMEDCT-NO-EXTENDED/concepts/${conceptId}`
  return sendQuerySnomedAPI(url)
}

function sendQuerySnomedAPI (url) {
  const response = httpClientLib.request({
    url: url,
    method: 'GET',
    contentType: 'application/json'
  })

  let ret
  try {
    ret = response.body && JSON.parse(response.body)
  } catch (e) {
    log.error('error parsing response from snowstorm')
  }
  return ret
}

Any progress here, @ase?

Important for us in order to succeed with terminology implementation in the Norwegian health service:)

Hi,

Sorry for the delayed response.
We have now solved the issue - at least I’m no longer getting the error locally with your code. The fix is coming in the next release of Content Studio.

NB! There are some duplicate Ids in response from the API you are using. For example, “Brokk” is listed twice with id 52515009. In this case CustomSelector will return the “Unique ID” error, even with our fix.

Thank you so much, @ase!! Very much appreciated!

The «brokk» is a bug on our side:) We will fix!

Do you know when the next release would be available?

Around mid-January, if everything goes according to the plan :slight_smile:

1 Like