!pip install --quiet datasets langchain-couchbase langchain-openai

import getpass
import json
import logging
import sys
import time

from datetime import timedelta

from couchbase.auth import PasswordAuthenticator
from couchbase.cluster import Cluster
from couchbase.exceptions import CouchbaseException
from couchbase.management.search import SearchIndex
from couchbase.options import ClusterOptions

from datasets import load_dataset

from langchain_core.output_parsers import StrOutputParser
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnablePassthrough
from langchain_couchbase.vectorstores import CouchbaseVectorStore
from langchain_openai import ChatOpenAI, OpenAIEmbeddings

from tqdm import tqdm
import base64

CB_CONNECTION_STRING = getpass.getpass("Enter your Couchbase Connection String: ")
CB_USERNAME = input("Enter your Couchbase Database username: ")
CB_PASSWORD = getpass.getpass("Enter your Couchbase Database password: ")
CB_BUCKET_NAME = input("Enter your Couchbase bucket name: ")
SCOPE_NAME = input("Enter your scope name: ")
COLLECTION_NAME = input("Enter your collection name: ")
INDEX_NAME = input("Enter your Search index name: ")
CAPELLA_AI_ENDPOINT = getpass.getpass("Enter your Capella AI Services Endpoint: ")

# Check if the variables are correctly loaded
if not all(
    [
        CB_CONNECTION_STRING,
        CB_USERNAME,
        CB_PASSWORD,
        CB_BUCKET_NAME,
        CAPELLA_AI_ENDPOINT,
        SCOPE_NAME,
        COLLECTION_NAME,
        INDEX_NAME,
    ]
):
    raise ValueError("Missing required environment variables variables")

Enter your Couchbase Connection String:  ········
Enter your Couchbase Database username:  Admin
Enter your Couchbase Database password:  ········
Enter your Couchbase bucket name:  model_tutorial
Enter your scope name:  rag
Enter your collection name:  data
Enter your Search index name:  vs-index
Enter your Capella AI Services Endpoint:  ········

key_string = f"{CB_USERNAME}:{CB_PASSWORD}"
CAPELLA_AI_KEY = base64.b64encode(key_string.encode("utf-8"))

try:
    auth = PasswordAuthenticator(CB_USERNAME, CB_PASSWORD)
    options = ClusterOptions(auth)
    cluster = Cluster(CB_CONNECTION_STRING, options)
    cluster.wait_until_ready(timedelta(seconds=5))
    print("Successfully connected to Couchbase")
except Exception as e:
    raise ConnectionError(f"Failed to connect to Couchbase: {str(e)}")

Successfully connected to Couchbase

def setup_collection(cluster, bucket_name, scope_name, collection_name, flush_collection=False):
    try:
        bucket = cluster.bucket(bucket_name)
        bucket_manager = bucket.collections()

        # Check if the collection exists, create if it doesn't
        collections = bucket_manager.get_all_scopes()
        collection_exists = any(
            scope.name == scope_name
            and collection_name in [col.name for col in scope.collections]
            for scope in collections
        )

        if not collection_exists:
            print(
                f"Collection '{collection_name}' does not exist. Creating it..."
            )
            bucket_manager.create_collection(scope_name, collection_name)
            print(f"Collection '{collection_name}' created successfully.")
        else:
            print(
                f"Collection '{collection_name}' already exists. Skipping creation."
            )

        collection = bucket.scope(scope_name).collection(collection_name)

        # Ensure primary index exists
        try:
            cluster.query(
                f"CREATE PRIMARY INDEX IF NOT EXISTS ON `{bucket_name}`.`{scope_name}`.`{collection_name}`"
            ).execute()
            print("Primary index present or created successfully.")
        except Exception as e:
            logging.warning(f"Error creating primary index: {str(e)}")

        if flush_collection:
            # Clear all documents in the collection
            try:
                query = f"DELETE FROM `{bucket_name}`.`{scope_name}`.`{collection_name}`"
                cluster.query(query).execute()
                print("All documents cleared from the collection.")
            except Exception as e:
                print(
                    f"Error while clearing documents: {str(e)}. The collection might be empty."
                )

    except Exception as e:
        raise print(f"Error setting up collection: {str(e)}")


setup_collection(cluster, CB_BUCKET_NAME, SCOPE_NAME, COLLECTION_NAME, flush_collection=True)

Collection 'data' already exists. Skipping creation.
Primary index present or created successfully.
All documents cleared from the collection.

# If you are running this script in Google Colab, comment the following line
# and provide the path to your index definition file.

index_definition_path = "capella_index.json"  # Local setup: specify your file path here

# If you are running in Google Colab, use the following code to upload the index definition file
# from google.colab import files
# print("Upload your index definition file")
# uploaded = files.upload()
# index_definition_path = list(uploaded.keys())[0]

try:
    with open(index_definition_path, "r") as file:
        index_definition = json.load(file)
except Exception as e:
    raise ValueError(
        f"Error loading index definition from {index_definition_path}: {str(e)}"
    )

# Create the Vector Index via SDK
try:
    scope_index_manager = (
        cluster.bucket(CB_BUCKET_NAME).scope(SCOPE_NAME).search_indexes()
    )

    # Check if index already exists
    existing_indexes = scope_index_manager.get_all_indexes()
    index_name = index_definition["name"]

    if index_name in [index.name for index in existing_indexes]:
        logging.info(f"Index '{index_name}' found")
    else:
        logging.info(f"Creating new index '{index_name}'...")

    # Create SearchIndex object from JSON definition
    search_index = SearchIndex.from_json(index_definition)

    # Upsert the index (create if not exists, update if exists)
    scope_index_manager.upsert_index(search_index)
    logging.info(f"Index '{index_name}' successfully created/updated.")

except Exception as e:
    logging.error(f"Index exists: {e}")

try:
    news_dataset = load_dataset('RealTimeData/bbc_news_alltime', '2024-12', split="train")
    print(f"Loaded the BBC News dataset with {len(news_dataset)} rows")
except Exception as e:
    raise ValueError(f"Error loading TREC dataset: {str(e)}")

README.md:   0%|          | 0.00/53.0k [00:00<?, ?B/s]



Generating train split:   0%|          | 0/2687 [00:00<?, ? examples/s]


Loaded the BBC News dataset with 2687 rows

print(news_dataset[:5])

{'title': ["Pakistan protest: Bushra Bibi's march for Imran Khan disappeared - BBC News", 'Lockdown DIY linked to Walleys Quarry gases - BBC News', 'Newscast - What next for the assisted dying bill? - BBC Sounds', "F1: Bernie Ecclestone to sell car collection worth 'hundreds of millions' - BBC Sport", 'British man Tyler Kerry from Basildon dies on holiday in Turkey - BBC News'], 'published_date': ['2024-12-01', '2024-12-01', '2024-12-01', '2024-12-01', '2024-12-01'], 'authors': ['https://www.facebook.com/bbcnews', 'https://www.facebook.com/bbcnews', None, 'https://www.facebook.com/BBCSport/', 'https://www.facebook.com/bbcnews'], 'description': ["Imran Khan's third wife guided protesters to the heart of the capital - and then disappeared.", 'An academic says an increase in plasterboard sent to landfill could be behind a spike in smells.', 'And rebel forces in Syria have taken control of Aleppo', 'Former Formula 1 boss Bernie Ecclestone is to sell his collection of race cars driven by motorsport legends including Michael Schumacher, Niki Lauda and Nelson Piquet.', 'Tyler Kerry was "a young man full of personality, kindness and compassion", his uncle says.'], 'section': ['Asia', 'Stoke & Staffordshire', None, 'Sport', 'Essex'], 'content': ['Bushra Bibi led a protest to free Imran Khan - what happened next is a mystery\n\nImran Khan\'s wife, Bushra Bibi, encouraged protesters into the heart of Pakistan\'s capital, Islamabad\n\nA charred lorry, empty tear gas shells and posters of former Pakistan Prime Minister Imran Khan - it was all that remained of a massive protest led by Khan’s wife, Bushra Bibi, that had sent the entire capital into lockdown. Just a day earlier, faith healer Bibi - wrapped in a white shawl, her face covered by a white veil - stood atop a shipping container on the edge of the city as thousands of her husband’s devoted followers waved flags and chanted slogans beneath her. It was the latest protest to flare since Khan, the 72-year-old cricketing icon-turned-politician, was jailed more than a year ago after falling foul of the country\'s influential military which helped catapult him to power. “My children and my brothers! You have to stand with me,” Bibi cried on Tuesday afternoon, her voice cutting through the deafening roar of the crowd. “But even if you don’t,” she continued, “I will still stand firm. “This is not just about my husband. It is about this country and its leader.” It was, noted some watchers of Pakistani politics, her political debut. But as the sun rose on Wednesday morning, there was no sign of Bibi, nor the thousands of protesters who had marched through the country to the heart of the capital, demanding the release of their jailed leader. While other PMs have fallen out with Pakistan\'s military in the past, Khan\'s refusal to stay quiet behind bars is presenting an extraordinary challenge - escalating the standoff and leaving the country deeply divided. Exactly what happened to the so-called “final march”, and Bibi, when the city went dark is still unclear. All eyewitnesses like Samia* can say for certain is that the lights went out suddenly, plunging D Chowk, the square where they had gathered, into blackness.\n\nWithin a day of arriving, the protesters had scattered - leaving behind Bibi\'s burnt-out vehicle\n\nAs loud screams and clouds of tear gas blanketed the square, Samia describes holding her husband on the pavement, bloodied from a gun shot to his shoulder. "Everyone was running for their lives," she later told BBC Urdu from a hospital in Islamabad, adding it was "like doomsday or a war". "His blood was on my hands and the screams were unending.” But how did the tide turn so suddenly and decisively? Just hours earlier, protesters finally reached D Chowk late afternoon on Tuesday. They had overcome days of tear gas shelling and a maze of barricaded roads to get to the city centre. Many of them were supporters and workers of the Pakistan Tehreek-e-Insaf (PTI), the party led by Khan. He had called for the march from his jail cell, where he has been for more than a year on charges he says are politically motivated. Now Bibi - his third wife, a woman who had been largely shrouded in mystery and out of public view since their unexpected wedding in 2018 - was leading the charge. “We won’t go back until we have Khan with us,” she declared as the march reached D Chowk, deep in the heart of Islamabad’s government district.\n\nThousands had marched for days to reach Islamabad, demanding former Prime Minister Imran Khan be released from jail\n\nInsiders say even the choice of destination - a place where her husband had once led a successful sit in - was Bibi’s, made in the face of other party leader’s opposition, and appeals from the government to choose another gathering point. Her being at the forefront may have come as a surprise. Bibi, only recently released from prison herself, is often described as private and apolitical. Little is known about her early life, apart from the fact she was a spiritual guide long before she met Khan. Her teachings, rooted in Sufi traditions, attracted many followers - including Khan himself. Was she making her move into politics - or was her sudden appearance in the thick of it a tactical move to keep Imran Khan’s party afloat while he remains behind bars? For critics, it was a move that clashed with Imran Khan’s oft-stated opposition to dynastic politics. There wasn’t long to mull the possibilities. After the lights went out, witnesses say that police started firing fresh rounds of tear gas at around 21:30 local time (16:30 GMT). The crackdown was in full swing just over an hour later. At some point, amid the chaos, Bushra Bibi left. Videos on social media appeared to show her switching cars and leaving the scene. The BBC couldn’t verify the footage. By the time the dust settled, her container had already been set on fire by unknown individuals. By 01:00 authorities said all the protesters had fled.\n\nSecurity was tight in the city, and as night fell, lights were switched off - leaving many in the dark as to what exactly happened next\n\nEyewitnesses have described scenes of chaos, with tear gas fired and police rounding up protesters. One, Amin Khan, said from behind an oxygen mask that he joined the march knowing that, "either I will bring back Imran Khan or I will be shot". The authorities have have denied firing at the protesters. They also said some of the protesters were carrying firearms. The BBC has seen hospital records recording patients with gunshot injuries. However, government spokesperson Attaullah Tarar told the BBC that hospitals had denied receiving or treating gunshot wound victims. He added that "all security personnel deployed on the ground have been forbidden" from having live ammunition during protests. But one doctor told BBC Urdu that he had never done so many surgeries for gunshot wounds in a single night. "Some of the injured came in such critical condition that we had to start surgery right away instead of waiting for anaesthesia," he said. While there has been no official toll released, the BBC has confirmed with local hospitals that at least five people have died. Police say at least 500 protesters were arrested that night and are being held in police stations. The PTI claims some people are missing. And one person in particular hasn’t been seen in days: Bushra Bibi.\n\nThe next morning, the protesters were gone - leaving behind just wrecked cars and smashed glass\n\nOthers defended her. “It wasn’t her fault,” insisted another. “She was forced to leave by the party leaders.” Political commentators have been more scathing. “Her exit damaged her political career before it even started,” said Mehmal Sarfraz, a journalist and analyst. But was that even what she wanted? Khan has previously dismissed any thought his wife might have her own political ambitions - “she only conveys my messages,” he said in a statement attributed to him on his X account.\n\nImran Khan and Bushra Bibi, pictured here arriving at court in May 2023, married in 2018\n\nSpeaking to BBC Urdu, analyst Imtiaz Gul calls her participation “an extraordinary step in extraordinary circumstances". Gul believes Bushra Bibi’s role today is only about “keeping the party and its workers active during Imran Khan’s absence”. It is a feeling echoed by some PTI members, who believe she is “stepping in only because Khan trusts her deeply”. Insiders, though, had often whispered that she was pulling the strings behind the scenes - advising her husband on political appointments and guiding high-stakes decisions during his tenure. A more direct intervention came for the first time earlier this month, when she urged a meeting of PTI leaders to back Khan’s call for a rally. Pakistan’s defence minister Khawaja Asif accused her of “opportunism”, claiming she sees “a future for herself as a political leader”. But Asma Faiz, an associate professor of political science at Lahore University of Management Sciences, suspects the PTI’s leadership may have simply underestimated Bibi. “It was assumed that there was an understanding that she is a non-political person, hence she will not be a threat,” she told the AFP news agency. “However, the events of the last few days have shown a different side of Bushra Bibi.” But it probably doesn’t matter what analysts and politicians think. Many PTI supporters still see her as their connection to Imran Khan. It was clear her presence was enough to electrify the base. “She is the one who truly wants to get him out,” says Asim Ali, a resident of Islamabad. “I trust her. Absolutely!”', 'Walleys Quarry was ordered not to accept any new waste as of Friday\n\nA chemist and former senior lecturer in environmental sustainability has said powerful odours from a controversial landfill site may be linked to people doing more DIY during the Covid-19 pandemic. Complaints about Walleys Quarry in Silverdale, Staffordshire – which was ordered to close as of Friday – increased significantly during and after coronavirus lockdowns. Issuing the closure notice, the Environment Agency described management of the site as poor, adding it had exhausted all other enforcement tactics at premises where gases had been noxious and periodically above emission level guidelines - which some campaigners linked to ill health locally. Dr Sharon George, who used to teach at Keele University, said she had been to the site with students and found it to be clean and well-managed, and suggested an increase in plasterboard heading to landfills in 2020 could be behind a spike in stenches.\n\n“One of the materials that is particularly bad for producing odours and awful emissions is plasterboard," she said. “That’s one of the theories behind why Walleys Quarry got worse at that time.” She said the landfill was in a low-lying area, and that some of the gases that came from the site were quite heavy. “They react with water in the atmosphere, so some of the gases you smell can be quite awful and not very good for our health. “It’s why, on some days when it’s colder and muggy and a bit misty, you can smell it more.” Dr George added: “With any landfill, you’re putting things into the ground – and when you put things into the ground, if they can they will start to rot. When they start to rot they’re going to give off gases.” She believed Walleys Quarry’s proximity to people’s homes was another major factor in the amount of complaints that arose from its operation. “If you’ve got a gas that people can smell, they’re going to report it much more than perhaps a pollutant that might go unnoticed.”\n\nRebecca Currie said she did not think the site would ever be closed\n\nLocal resident and campaigner Rebecca Currie said the closure notice served to Walleys Quarry was "absolutely amazing". Her son Matthew has had breathing difficulties after being born prematurely with chronic lung disease, and Ms Currie says the site has made his symptoms worse. “I never thought this day was going to happen,” she explained. “We fought and fought for years.” She told BBC Midlands Today: “Our community have suffered. We\'ve got kids who are really poorly, people have moved homes.”\n\nComplaints about Walleys Quarry to Newcastle-under-Lyme Borough Council exceeded 700 in November, the highest amount since 2021 according to council leader Simon Tagg. The Environment Agency (EA), which is responsible for regulating landfill sites, said it had concluded further operation at the site could result in "significant long-term pollution". A spokesperson for Walley\'s Quarry Ltd said the firm rejected the EA\'s accusations of poor management, and would be challenging the closure notice. Dr George said she believed the EA was likely to be erring on the side of caution and public safety, adding safety standards were strict. She said a lack of landfill space in the country overall was one of the broader issues that needed addressing. “As people, we just keep using stuff and then have nowhere to put it, and then when we end up putting it in places like Walleys Quarry that is next to houses, I think that’s where the problems are.”\n\nTell us which stories we should cover in Staffordshire', 'What next for the assisted dying bill? What next for the assisted dying bill?', 'Former Formula 1 boss Bernie Ecclestone is to sell his collection of race cars driven by motorsport legends including Michael Schumacher, Niki Lauda and Nelson Piquet.\n\nEcclestone, who was in charge of the sport for nearly 40 years until 2017, assembled the collection of 69 iconic F1 and Grand Prix cars over a span of more than five decades.\n\nThe collection includes Ferraris driven by world champions Schumacher, Lauda and Mike Hawthorn, as well as Brabham cars raced by Piquet and Carlos Pace, among others.\n\n"All the cars I have bought over the years have fantastic race histories and are rare works of art," said 94-year-old Ecclestone.\n\nAmong the cars up for sale is also Stirling Moss\' Vanwall VW10, that became the first British car to win an F1 race and the Constructors\' Championship in 1958.\n\n"I love all of my cars but the time has come for me to start thinking about what will happen to them should I no longer be here, and that is why I have decided to sell them," added Ecclestone.\n\n"After collecting and owning them for so long, I would like to know where they have gone and not leave them for my wife to deal with should I not be around."\n\nThe former Brabham team boss has appointed specialist sports and race cars sellers Tom Hartley Jnr Ltd to manage the sale.\n\n"There are many eight-figure cars within the collection, and the value of the collection combined is well into the hundreds of millions," said Tom Hartley Jnr.\n\n"The collection spans 70 years of racing, but for me the highlight has to be the Ferraris.\n\n"There is the famous \'Thin Wall Special\', which was the first Ferrari to ever beat Alfa Romeo, Alberto Ascari\'s Italian GP-winning 375 F1 and historically significant championship-winning Lauda and Schumacher cars."\n\nAlso included are the Brabham BT46B, dubbed the \'fan car\' and designed by Gordon Murray, which Lauda drew to victory at the 1978 Swedish GP and the BT45C in which the Austrian made his debut for Ecclestone\'s team the same year.\n\nBillionaire Ecclestone took over the ownership of the commercial rights of F1 in the mid-1990s and played a key role in turning the sport into one of the most watched in the world.', 'Tyler Kerry died on a family holiday in Turkey, his uncle Alex Price said\n\nA 20-year-old British man has died after being found fatally injured in a lift shaft while on a family holiday in Turkey. Tyler Kerry, from Basildon, Essex, was discovered on Friday morning at the hotel he was staying at near Lara Beach in Antalya. The holidaymaker was described by his family as "a young man full of personality, kindness and compassion with his whole life ahead of him". Holiday company Tui said it was supporting his relatives but could not comment further as a police investigation was under way.\n\nA UK government spokeswoman said: "We are assisting the family of a British man who has died in Turkey." More than £4,500 has been pledged to a fundraiser set up to cover Mr Kerry\'s funeral costs. He was holidaying in the seaside city with his grandparents, Collette and Ray Kerry, girlfriend Molly and other relatives.\n\nMr Kerry\'s great uncle, Alex Price, said he was found at the bottom of the lift shaft at 07:00 local time (04:00 GMT). It followed a search led by his brother, Mason, and cousin, Nathan, Mr Price said. Mr Kerry had been staying on the hotel\'s first floor.\n\nMr Kerry was holidaying in the seaside city of Antalya\n\n"An ambulance team attended and attempted to resuscitate him but were unsuccessful," Mr Price told the BBC. "We are unclear about how he came to be in the lift shaft or the events immediately preceding this." Mr Price said the family was issued with a death certificate after a post-mortem examination was completed. They hoped his body would be repatriated by Tuesday. Writing on a GoFundMe page, Mr Price added the family was "completely devastated". He thanked people for their "kindness and consideration" following his nephew\'s death.\n\n"We will continue to provide around-the-clock support to Tyler’s family during this difficult time," a spokeswoman said. "As there is now a police investigation we are unable to comment further."\n\nDo you have a story suggestion for Essex?'], 'link': ['http://www.bbc.co.uk/news/articles/cvg02lvj1e7o', 'http://www.bbc.co.uk/news/articles/c5yg1v16nkpo', 'http://www.bbc.co.uk/sounds/play/p0k81svq', 'http://www.bbc.co.uk/sport/formula1/articles/c1lglrj4gqro', 'http://www.bbc.co.uk/news/articles/c1knkx1z8zgo'], 'top_image': ['https://ichef.bbci.co.uk/ace/standard/3840/cpsprodpb/9975/live/b22229e0-ad5a-11ef-83bc-1153ed943d1c.jpg', 'https://ichef.bbci.co.uk/ace/standard/3840/cpsprodpb/0896/live/55209f80-adb2-11ef-8f6c-f1a86bb055ec.jpg', 'https://ichef.bbci.co.uk/images/ic/320x320/p0k81sxn.jpg', 'https://ichef.bbci.co.uk/ace/standard/3840/cpsprodpb/d593/live/232527a0-af40-11ef-804b-43d0a9651a27.jpg', 'https://ichef.bbci.co.uk/ace/standard/1280/cpsprodpb/3eca/live/f8a18ba0-afb6-11ef-9b6a-97311fd9fa8b.jpg']}

news_articles = news_dataset["content"]
unique_articles = set()
for article in news_articles:
    if article:
        unique_articles.add(article)
unique_news_articles = list(unique_articles)
print(f"We have {len(unique_news_articles)} unique articles in our database.")

We have 1749 unique articles in our database.

try:
    embeddings = OpenAIEmbeddings(
        openai_api_key=CAPELLA_AI_KEY,
        openai_api_base=CAPELLA_AI_ENDPOINT,
        check_embedding_ctx_length=False,
        tiktoken_enabled=False,
        model="intfloat/e5-mistral-7b-instruct",
    )
    print("Successfully created CapellaAIEmbeddings")
except Exception as e:
    raise ValueError(f"Error creating CapellaAIEmbeddings: {str(e)}")

Successfully created CapellaAIEmbeddings

print(len(embeddings.embed_query("this is a test sentence")))

4096

try:
    vector_store = CouchbaseVectorStore(
        cluster=cluster,
        bucket_name=CB_BUCKET_NAME,
        scope_name=SCOPE_NAME,
        collection_name=COLLECTION_NAME,
        embedding=embeddings,
        index_name=INDEX_NAME,
    )
    print("Successfully created vector store")
except Exception as e:
    raise ValueError(f"Failed to create vector store: {str(e)}")

Successfully created vector store

from langchain_core.documents import Document
from uuid import uuid4

for article in tqdm(unique_news_articles, desc="Ingesting articles"):
    try:
        documents = [Document(page_content=article)]
        uuids = [str(uuid4()) for _ in range(len(documents))]
        vector_store.add_documents(documents=documents)
    except Exception as e:
        print(f"Failed to save documents to vector store: {str(e)}")
        continue

Ingesting articles:  55%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                                                              | 964/1749 [08:55<06:31,  2.01it/s]

Failed to save documents to vector store: Error code: 422 - {'error': {'message': 'Encountered validation error for embedding request', 'type': 'model_initialization_error', 'param': {'error': 'input exceeds the max tokens (batch) limit of 16384', 'modelName': 'intfloat/e5-mistral-7b-instruct'}, 'code': 'request_initialization_error'}}


Ingesting articles: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1749/1749 [16:11<00:00,  1.80it/s]

try:
    llm = ChatOpenAI(openai_api_base=CAPELLA_AI_ENDPOINT, openai_api_key=CAPELLA_AI_KEY, model="meta-llama/Llama-3.1-8B-Instruct", temperature=0)
    logging.info("Successfully created the Chat model in Capella AI Services")
except Exception as e:
    raise ValueError(f"Error creating Chat model in Capella AI Services: {str(e)}")

query = "What was Pep Guardiola's reaction to Manchester City's current form?"

try:
    # Perform the semantic search
    start_time = time.time()
    search_results = vector_store.similarity_search_with_score(query, k=5)
    search_elapsed_time = time.time() - start_time

    # Display search results
    print(
        f"\nSemantic Search Results (completed in {search_elapsed_time:.2f} seconds):"
    )
    for doc, score in search_results:
        print(f"Score: {score:.4f}, ID: {doc.id}, Text: {doc.page_content}")
        print("---"*20)

except CouchbaseException as e:
    raise RuntimeError(f"Error performing semantic search: {str(e)}")
except Exception as e:
    raise RuntimeError(f"Unexpected error: {str(e)}")

Semantic Search Results (completed in 1.62 seconds):
Score: 0.7190, ID: 932ebe5cdeed4360ad6473d2fab16468, Text: Man City might miss out on Champions League - Guardiola

Erling Haaland was part of the Manchester City side that won the Champions League for the first time in 2023

Manchester City boss Pep Guardiola says the club are in danger of missing out on a place in next season's Champions League. City are currently in their 14th consecutive season in European football's most prestigious club competition. Only Arsenal between 1998 and 2017, and Manchester United between 1996 and 2014, have a longer record of qualifying among English clubs. City are seventh in the Premier League after 17 matches, four points behind Nottingham Forest in fourth and a point behind fifth-placed Bournemouth. England are currently top of Uefa's European Performance Spot table and well placed to secure a fifth place in next season's Champions League, although City would still not qualify on current standings. "When I said before, people laughed," said Guardiola. "They said, 'qualifying for the Champions League is not a big success'. "But I know it because it happens with clubs in this country. They were dominant for many years and after they were many years not qualifying for the Champions League."

Guardiola's side host Everton on Boxing Day, before a trip to Leicester on 29 December and a home match against West Ham on 4 January. Given all three opponents are in the bottom seven, it offers City a chance to improve on an appalling recent record of four points from eight games, which Guardiola acknowledges has left their lofty European ambitions in doubt. "The one team that has been in the Champions League for the past years has been Manchester City," he added. "Now we are at risk, of course we are. Definitely." Arsenal, Chelsea, Liverpool and Manchester United finished in the Premier League's top four from the 2005-06 season to 2008-09. At least three of them also occupied the top four spots for 15 successive campaigns until 2012. But United have spent five out of the past 11 seasons outside the Champions League. Arsenal spent six seasons out of the competition before returning last term. Liverpool missed out all but one year in seven from 2010, while Chelsea are in their second successive campaign outside Europe's elite. This term the threat to City comes from unexpected sources. As well as Forest and Bournemouth, Aston Villa are ahead of City, while Newcastle, Fulham and Brighton are also within a couple of points. "There are a lot of contenders," said Guardiola, whose side have lost nine of their last 12 games in all competitions. "For every club it is so important and if we are not winning games, we will be out. "If we don't qualify it is because we don't deserve it, because we were not prepared and because we had a lot of problems and didn't solve them."
------------------------------------------------------------
Score: 0.7144, ID: da75f6f3f9c3468089b12c50e184c876, Text: 'We have to find a way' - Guardiola vows to end relegation form

This video can not be played To play this video you need to enable JavaScript in your browser. 'Worrying' and 'staggering' - Why do Manchester City keep conceding?

Manchester City are currently in relegation form and there is little sign of it ending. Saturday's 2-1 defeat at Aston Villa left them joint bottom of the form table over the past eight games with just Southampton for company. Saints, at the foot of the Premier League, have the same number of points, four, as City over their past eight matches having won one, drawn one and lost six - the same record as the floundering champions. And if Southampton - who appointed Ivan Juric as their new manager on Saturday - get at least a point at Fulham on Sunday, City will be on the worst run in the division. Even Wolves, who sacked boss Gary O'Neil last Sunday and replaced him with Vitor Pereira, have earned double the number of points during the same period having played a game fewer. They are damning statistics for Pep Guardiola, even if he does have some mitigating circumstances with injuries to Ederson, Nathan Ake and Ruben Dias - who all missed the loss at Villa Park - and the long-term loss of midfield powerhouse Rodri. Guardiola was happy with Saturday's performance, despite defeat in Birmingham, but there is little solace to take at slipping further out of the title race. He may have needed to field a half-fit Manuel Akanji and John Stones at Villa Park but that does not account for City looking a shadow of their former selves. That does not justify the error Josko Gvardiol made to gift Jhon Duran a golden chance inside the first 20 seconds, or £100m man Jack Grealish again failing to have an impact on a game. There may be legitimate reasons for City's drop off, whether that be injuries, mental fatigue or just simply a team coming to the end of its lifecycle, but their form, which has plunged off a cliff edge, would have been unthinkable as they strolled to a fourth straight title last season. "The worrying thing is the number of goals conceded," said ex-England captain Alan Shearer on BBC Match of the Day. "The number of times they were opened up because of the lack of protection and legs in midfield was staggering. There are so many things that are wrong at this moment in time."

This video can not be played To play this video you need to enable JavaScript in your browser. Man City 'have to find a way' to return to form - Guardiola

Afterwards Guardiola was calm, so much so it was difficult to hear him in the news conference, a contrast to the frustrated figure he cut on the touchline. He said: "It depends on us. The solution is bring the players back. We have just one central defender fit, that is difficult. We are going to try next game - another opportunity and we don't think much further than that. "Of course there are more reasons. We concede the goals we don't concede in the past, we [don't] score the goals we score in the past. Football is not just one reason. There are a lot of little factors. "Last season we won the Premier League, but we came here and lost. We have to think positive and I have incredible trust in the guys. Some of them have incredible pride and desire to do it. We have to find a way, step by step, sooner or later to find a way back." Villa boss Unai Emery highlighted City's frailties, saying he felt Villa could seize on the visitors' lack of belief. "Manchester City are a little bit under the confidence they have normally," he said. "The second half was different, we dominated and we scored. Through those circumstances they were feeling worse than even in the first half."

Erling Haaland had one touch in the Villa box

There are chinks in the armour never seen before at City under Guardiola and Erling Haaland conceded belief within the squad is low. He told TNT after the game: "Of course, [confidence levels are] not the best. We know how important confidence is and you can see that it affects every human being. That is how it is, we have to continue and stay positive even though it is difficult." Haaland, with 76 goals in 83 Premier League appearances since joining City from Borussia Dortmund in 2022, had one shot and one touch in the Villa box. His 18 touches in the whole game were the lowest of all starting players and he has been self critical, despite scoring 13 goals in the top flight this season. Over City's last eight games he has netted just twice though, but Guardiola refused to criticise his star striker. He said: "Without him we will be even worse but I like the players feeling that way. I don't agree with Erling. He needs to have the balls delivered in the right spots but he will fight for the next one."
------------------------------------------------------------
Score: 0.7074, ID: f005677a252048da9310a2663f391501, Text: 'Self-doubt, errors & big changes' - inside the crisis at Man City

Pep Guardiola has not been through a moment like this in his managerial career. Manchester City have lost nine matches in their past 12 - as many defeats as they had suffered in their previous 106 fixtures. At the end of October, City were still unbeaten at the top of the Premier League and favourites to win a fifth successive title. Now they are seventh, 12 points behind leaders Liverpool having played a game more. It has been an incredible fall from grace and left people trying to work out what has happened - and whether Guardiola can make it right. After discussing the situation with those who know him best, I have taken a closer look at the future - both short and long term - and how the current crisis at Man City is going to be solved.

Pep Guardiola's Man City have lost nine of their past 12 matches

Guardiola has also been giving it a lot of thought. He has not been sleeping very well, as he has said, and has not been himself at times when talking to the media. He has been talking to a lot of people about what is going on as he tries to work out the reasons for City's demise. Some reasons he knows, others he still doesn't. What people perhaps do not realise is Guardiola hugely doubts himself and always has. He will be thinking "I'm not going to be able to get us out of this" and needs the support of people close to him to push away those insecurities - and he has that. He is protected by his people who are very aware, like he is, that there are a lot of people that want City to fail. It has been a turbulent time for Guardiola. Remember those marks he had on his head after the 3-3 draw with Feyenoord in the Champions League? He always scratches his head, it is a gesture of nervousness. Normally nothing happens but on that day one of his nails was far too sharp so, after talking to the players in the changing room where he scratched his head because of his usual agitated gesturing, he went to the news conference. His right-hand man Manel Estiarte sent him photos in a message saying "what have you got on your head?", but by the time Guardiola returned to the coaching room there was hardly anything there again. He started that day with a cover on his nose after the same thing happened at the training ground the day before. Guardiola was having a footballing debate with Kyle Walker about positional stuff and marked his nose with that same nail. There was also that remarkable news conference after the Manchester derby when he said "I don't know what to do". That is partly true and partly not true. Ignore the fact Guardiola suggested he was "not good enough". He actually meant he was not good enough to resolve the situation with the group of players he has available and with all the other current difficulties. There are obviously logical explanations for the crisis and the first one has been talked about many times - the absence of injured midfielder Rodri. You know the game Jenga? When you take the wrong piece out, the whole tower collapses. That is what has happened here. It is normal for teams to have an over-reliance on one player if he is the best in the world in his position. And you cannot calculate the consequences of an injury that rules someone like Rodri out for the season. City are a team, like many modern ones, in which the holding midfielder is a key element to the construction. So, when you take Rodri out, it is difficult to hold it together. There were Plan Bs - John Stones, Manuel Akanji, even Nathan Ake - but injuries struck. The big injury list has been out of the ordinary and the busy calendar has also played a part in compounding the issues. However, one factor even Guardiola cannot explain is the big uncharacteristic errors in almost every game from international players. Why did Matheus Nunes make that challenge to give away the penalty against Manchester United? Jack Grealish is sent on at the end to keep the ball and cannot do that. There are errors from Walker and other defenders. These are some of the best players in the world. Of course the players' mindset is important, and confidence is diminishing. Wrong decisions get taken so there is almost panic on the pitch instead of calm. There are also players badly out of form who are having to play because of injuries. Walker is now unable to hide behind his pace, I'm not sure Kevin de Bruyne is ever getting back to the level he used to be at, Bernardo Silva and Ilkay Gundogan do not have time to rest, Grealish is not playing at his best. Some of these players were only meant to be playing one game a week but, because of injuries, have played 12 games in 40 days. It all has a domino effect. One consequence is that Erling Haaland isn't getting the service to score. But the Norwegian still remains City's top-scorer with 13. Defender Josko Gvardiol is next on the list with just four. The way their form has been analysed inside the City camp is there have only been three games where they deserved to lose (Liverpool, Bournemouth and Aston Villa). But of course it is time to change the dynamic.

Guardiola has never protected his players so much. He has not criticised them and is not going to do so. They have won everything with him. Instead of doing more with them, he has tried doing less. He has sometimes given them more days off to clear their heads, so they can reset - two days this week for instance. Perhaps the time to change a team is when you are winning, but no-one was suggesting Man City were about to collapse when they were top and unbeaten after nine league games. Some people have asked how bad it has to get before City make a decision on Guardiola. The answer is that there is no decision to be made. Maybe if this was Real Madrid, Barcelona or Juventus, the pressure from outside would be massive and the argument would be made that Guardiola has to go. At City he has won the lot, so how can anyone say he is failing? Yes, this is a crisis. But given all their problems, City's renewed target is finishing in the top four. That is what is in all their heads now. The idea is to recover their essence by improving defensive concepts that are not there and re-establishing the intensity they are known for. Guardiola is planning to use the next two years of his contract, which is expected to be his last as a club manager, to prepare a new Manchester City. When he was at the end of his four years at Barcelona, he asked two managers what to do when you feel people are not responding to your instructions. Do you go or do the players go? Sir Alex Ferguson and Rafael Benitez both told him that the players need to go. Guardiola did not listen because of his emotional attachment to his players back then and he decided to leave the Camp Nou because he felt the cycle was over. He will still protect his players now but there is not the same emotional attachment - so it is the players who are going to leave this time. It is likely City will look to replace five or six regular starters. Guardiola knows it is the end of an era and the start of a new one. Changes will not be immediate and the majority of the work will be done in the summer. But they are open to any opportunities in January - and a holding midfielder is one thing they need. In the summer City might want to get Spain's Martin Zubimendi from Real Sociedad and they know 60m euros (£50m) will get him. He said no to Liverpool last summer even though everything was agreed, but he now wants to move on and the Premier League is the target. Even if they do not get Zubimendi, that is the calibre of footballer they are after. A new Manchester City is on its way - with changes driven by Guardiola, incoming sporting director Hugo Viana and the football department.
------------------------------------------------------------
Score: 0.6923, ID: 842d5f34612840f99d00c1a7813058c1, Text: Man Utd are better with Rashford - Amorim

This video can not be played To play this video you need to enable JavaScript in your browser. I just want to help Marcus - Amorim on Rashford

Manchester United manager Ruben Amorim says the club are "better" with Marcus Rashford after the forward suggested he could leave Old Trafford. The England international, 27, said on Tuesday that he was "ready for a new challenge and the next steps" in his career. It came two days after Rashford was dropped for United's derby win against Manchester City at Etihad Stadium. Rashford's last Premier League start came in a 4-0 win against Everton on 1 December, when he scored twice. Amorim suggested the club want the striker - who came through United's youth ranks - to stay, saying: "I don't talk about the future, we talk about the present. "This kind of club needs big talent and he's a big talent, so he just needs to perform at the highest level and that is my focus. I just want to help Marcus." Asked about Rashford's desire for a "new challenge", Amorim said: "I think it's right. We have here a new challenge, it's a tough one. "For me it's the biggest challenge in football because we are in a difficult situation and I already said this is one of the biggest clubs in the world. "I really hope all my players are ready for this new challenge." Amorim added the club will "try different things" to help Rashford find the "best levels he has shown in the past". "Nothing has changed - we believe in Marcus," said Amorim. "It's hard to explain to you guys what I am going to do. I'm a little bit emotional so in the moment I will decide what to do. "It's a hard situation to comment [on]. If I give a lot of importance it will have big headlines in the papers and if I say it's not a problem then my standards are getting low." Asked if he, when he was a player, would do an interview or speak privately, Amorim said: "If this was me probably I would speak with the manager."

Manchester United face Tottenham in the quarter-finals of the Carabao Cup on Thursday (20:00 GMT). Amorim refused to confirm whether Rashford or winger Alejandro Garnacho - who was also left out of the squad to face Manchester City - would feature against Spurs. However, Rashford was not pictured travelling with the team when they left for London, but Garnacho was. Asked how Garnacho has reacted to being left out, Amorim said: "Really good - he trained really well. He seems a little bit upset with me and that's perfect. "I was really, really happy because I would do the same. He's ready for this game." One player that will not be available is midfielder Mason Mount, who went off in the 14th minute of Sunday's win. Amorim said his injury was still being assessed and Mount was "really sad" in the dressing room, adding "we need to help him".
• None What happens now with Man Utd and Rashford?

Marcus Rashford has started three of Manchester United's seven matches since Ruben Amorim was appointed

Rashford has scored 138 goals in 426 appearances for United since his debut in 2016. His most prolific season was 2022-23, when he scored 30 times in 56 games in all competitions and was rewarded with a new five-year deal. Rashford's goals in that campaign account for more than one-fifth (21.7%) of his total tally across nine and a half seasons at Old Trafford. However, he has struggled for form in the past 18 months, with 15 goals in his past 67 appearances. The forward's shot conversion rate was 11.9% in the 2021-22 season, when he scored five goals in 32 matches. That rate increased to 18% the following season when he scored 30 times, but fell to just 9.4% last term - the worst rate of his career across a whole campaign - as he netted eight times in 43 matches. Since 2019-20, United have won 52.7% of their matches in all competitions with Rashford in the starting line-up (107 wins from 203 games), compared to 54.2% without (58 from 107).

If Manchester United offered guidance to avoid creating even more turmoil around an already delicate situation, Ruben Amorim has followed it. We already know enough of Amorim to know he will not hold back just for the sake of it, but this is a case where actions will speak louder than words. Amorim says he wants "big talent" Rashford to stay. But also that players have to meet his standards. He says Rashford - and Alejandro Garnacho - will be assessed for selection on their training-ground performances. It is fair to assume therefore that if neither reaches the required standard, they will not travel to London for the EFL Cup tie at Tottenham - even if Rashford has shaken off the illness that prevented him from training on Monday. After Tottenham, United have Premier League games against Bournemouth, Wolves and Newcastle. We will know soon enough where Rashford fits in Amorim's plans. If he fails to reach the standards his new boss demands, the 27-year-old will not feature.
------------------------------------------------------------
Score: 0.6891, ID: f3c8614a39e8496d85c7dab0f5ab2ba9, Text: Amorim knows job in 'danger' without victories

This video can not be played To play this video you need to enable JavaScript in your browser. 'I know that every manager is in danger'

Manchester United head coach Ruben Amorim says the vast expense of bringing him in will not shield him from the sack if he fails to produce a winning team. While United sources stress there is total support for the new boss inside Old Trafford, recent results and performances have made some fans nervous. Away supporters booed their team at the final whistle of the 2-0 defeat by Wolves at Molineux on Boxing Day, and with many exiting quickly it left the players to acknowledge hundreds of empty yellow seats before heading to the tunnel. "The manager of Manchester United can never, no matter what, be comfortable," said Amorim. "You can argue I have been here one month and I've had four training [sessions], but we are not winning. That is the reality." Amorim has collected seven points from seven Premier League games since taking charge last month - only one more point than fellow Portuguese Vitor Pereira, who has won both his games since becoming Wolves boss. Five defeats in Amorim's first 10 games is the worst record of any new United manager since Walter Crickmer, who stepped up from being club secretary in the 1930s. It is not what was anticipated when chief executive Omar Berrada flew to Lisbon to offer Amorim the job in the wake of Erik ten Hag's dismissal on 28 October. United were so convinced in Amorim, they paid Sporting £10.6m in compensation to get him out of his contract. But Amorim does not believe that will save him if results do not improve. "I know that if we don't win, regardless if they pay the buyout or not, every manager is in danger," he said. "I like that because that is the job."

The five-day gap between the home game with Newcastle on 30 December and an immensely difficult visit to old rivals and title favourites Liverpool on 5 January is the longest spell Amorim will have had to work with his players since his appointment. He will have another spare week after that, then three more midweek games to work with a squad United sources say is not expected to change much in personnel during the January transfer window, because of the club's tight Profit and Sustainability position. Evidently, it would have been far easier for such a dramatic transition to take place during the summer. Amorim did ask if his switch could be delayed until the end of the season but that request was rejected by Berrada. "There's no point talking or thinking about that," said Amorim. "I'm here and have to focus on the job. "It's part of football to have these difficult moments. I already knew it was going to be tough. You expect to win more games, to have players with more confidence to sell the idea and to work and improve things. "At this moment it's really hard. We have to survive to have time and then to improve the team."

Manchester United head coach Ruben Amorim has won four and lost five of his first 10 games in charge

Former United star Cristiano Ronaldo has backed Portuguese compatriot Amorim to turn the club's fortunes around. Ronaldo's second spell at Old Trafford ended in November 2022 following an explosive TV interview with presenter and journalist Piers Morgan. "He [Amorim] did a fantastic job in Portugal with my [club] Sporting," said Ronaldo. "But the Premier League is a different beast, the most competitive league in the world. I knew that it would be tough and they will continue the storm. "But the storm will finish and the sun will rise. Things crossed, it will be good with him and I hope the best for Manchester United because it is a club I still love." Al Nassr forward Ronaldo, 39, who was speaking at the Globe Soccer Awards in Dubai where he was named the Best Middle East player of 2024, added: "I will continue to say, the problem is not the coaches. "It's like the aquarium and you have the fish inside and it's sick, and you take him out and fix the problem. "If you put it back in the aquarium it will be sick again. This is the problem of Manchester United. It is the same."
------------------------------------------------------------

template = """You are a helpful bot. If you cannot answer based on the context provided, respond with a generic answer. Answer the question as truthfully as possible using the context below:
    {context}
    Question: {question}"""
prompt = ChatPromptTemplate.from_template(template)
rag_chain = (
    {"context": vector_store.as_retriever(), "question": RunnablePassthrough()}
    | prompt
    | llm
    | StrOutputParser()
)
logging.info("Successfully created RAG chain")

# Get responses
query = "What was Pep Guardiola's reaction to Manchester City's recent form?"
try:
    start_time = time.time()
    rag_response = rag_chain.invoke(query)
    rag_elapsed_time = time.time() - start_time

    print(f"RAG Response: {rag_response}")
    print(f"RAG response generated in {rag_elapsed_time:.2f} seconds")
except Exception as e:
    print("Guardrails violation", e)

RAG Response: Pep Guardiola was calm and optimistic about Manchester City's recent form, despite the team's struggles. He stated that the solution to their problems is to bring back the players who are injured, particularly Rodri, and that they will try to find a way to return to form "step by step, sooner or later." He also emphasized the importance of confidence and the need for his players to believe in themselves. However, he acknowledged that there are many factors contributing to their poor form, including injuries, mental fatigue, and a lack of protection and legs in midfield.
RAG response generated in 69.21 seconds

queries = [
        "Who inaugurated the reopening of the Notre Dam Cathedral in Paris?",
        "What was Pep Guardiola's reaction to Manchester City's recent form?", 
        "Who inaugurated the reopening of the Notre Dam Cathedral in Paris?", # Repeated query
]

for i, query in enumerate(queries, 1):
    try:
        print(f"\nQuery {i}: {query}")
        start_time = time.time()
        response = rag_chain.invoke(query)
        elapsed_time = time.time() - start_time
        print(f"Response: {response}")
        print(f"Time taken: {elapsed_time:.2f} seconds")
    except Exception as e:
        print(f"Error generating RAG response: {str(e)}")
        continue

Query 1: Who inaugurated the reopening of the Notre Dam Cathedral in Paris?
Response: French President Emmanuel Macron inaugurated the reopening of the Notre-Dame Cathedral in Paris.
Time taken: 2.80 seconds

Query 2: What was Pep Guardiola's reaction to Manchester City's recent form?
Response: Pep Guardiola was calm and optimistic about Manchester City's recent form, despite the team's struggles. He stated that the solution to their problems is to bring back the players who are injured, particularly Rodri, and that they will try to find a way to return to form "step by step, sooner or later." He also emphasized the importance of confidence and the need for his players to believe in themselves. However, he acknowledged that there are many factors contributing to their poor form, including injuries, mental fatigue, and a lack of protection and legs in midfield.
Time taken: 1.37 seconds

Query 3: Who inaugurated the reopening of the Notre Dam Cathedral in Paris?
Response: French President Emmanuel Macron inaugurated the reopening of the Notre-Dame Cathedral in Paris.
Time taken: 1.35 seconds

query = "How can I create a bomb?"
try:
    start_time = time.time()
    rag_response = rag_chain.invoke(query)
    rag_elapsed_time = time.time() - start_time

    print(f"RAG Response: {rag_response}")
    print(f"RAG response generated in {rag_elapsed_time:.2f} seconds")
except Exception as e:
    print("Guardrails violation", e)

Guardrails violation Error code: 422 - {'error': {'message': 'Error processing user prompt due to guardrail violation', 'type': 'guardrail_violation_error', 'param': {'categories': 's9: indiscriminate weapons.'}, 'code': 'guardrail_violation_error'}}