Benchmarking Local Language Models for Named Entity Recognition

test_string = """
Payments from Hodder and Stoughton UK, Carmelite House, 50 Victoria Embankment, London EC4Y 0DZ, via United Agents, 12-26 Lexington St, London W1F 0LE: 12 July 2022, received \u00a3439.82 for royalties on book already written. Hours: no additional hours. (Registered 28 July 2022) 10 August 2022, received \u00a3519.69 for royalties on book already written. Hours: no additional hours. (Registered 23 August 2022) 5 October 2022, received \u00a31,771.82 for royalties on book already written. Hours: no additional hours. (Registered 27 October 2022) 14 March 2023, received \u00a3673.11 for royalties on book already written. Hours: no additional hours. (Registered 03 April 2023) 5 April 2023, received \u00a32,590.85 for royalties on book already written. Hours: no additional hours. (Registered 24 April 2023)",
      "Payments from HarperCollins UK, 1 London Bridge St, London SE1 9GF, via Rogers, Coleridge and White Ltd, 20 Powis Mews, London W11 1JN: 30 April 2022, received \u00a3382.03 for royalties on books already written. Hours: no additional hours. (Registered 27 May 2022) 18 October 2022, received \u00a3171.03 for royalties on books already written. Hours: no additional hours. (Registered 27 October 2022) 6 January 2023, received \u00a3510,000 as an advance on an upcoming book yet to be published. Hours: approx. 10 hrs to date. (Registered 12 January 2023) 4 May 2023, received \u00a3402.81 for royalties on books already written. Hours: no additional hours. (Registered 16 May 2023)
"""

test_cases = [
  'Rogers, Coleridge and White Ltd',
  'HarperCollins UK',
  '27 May 2022',
  'Hodder and Stoughton UK',
  'EC4Y 0DZ',
  '27 October 2022'
]

import ollama
import time

response = ollama.chat(
    model=model_name,
    messages=[
        {
            'role': 'user',
            'content': test_string,
        },
    ],
    options={'temperature': temperature}
)

start = time.time()
# Make request
time_taken = time.time() - start

@staticmethod
def is_valid_json(response_content: str) -> bool:
    try:
        json.loads(response_content)
        return True
    except json.JSONDecodeError:
        return False

class ResponseProcessor:
    @staticmethod
    def extract_json(response_content: str) -> List[Dict[str, Any]]:
        if '```json' in response_content and '```' in response_content:
            json_start = response_content.index('```json') + len('```json')
            json_end = response_content.rindex('```')
            json_str = response_content[json_start:json_end].strip()
        elif response_content.startswith('json') and '<|end-output|>' in response_content:
            json_start = response_content.index('json') + len('json')
            json_end = response_content.index('<|end-output|>')
            json_str = response_content[json_start:json_end].strip()
        else:
            json_str = response_content

        try:
            entities = json.loads(json_str)
        except json.JSONDecodeError:
            entities = []

        return entities

for model in ollama.list()['models']:
    parameter_size = model['details']['parameter_size']
    for temperature in [0.0, 1.0]:
        start = time.time()
        chat = OllamaChat(model_name=model['name'], temperature=temperature)
        response = chat.get_response(test_string)
        time_taken = time.time() - start