Chunking Strategies for Optimizing Large Language Models (LLMs)

from langchain.text_splitter import RecursiveCharacterTextSplitter

# Function to split text with fixed-size chunks and overlap
def split_text_with_overlap(text, chunk_size, overlap_size):
    # Create a text splitter with overlap
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=chunk_size, 
        chunk_overlap=overlap_size
    )
    
    # Split the text
    chunks = text_splitter.split_text(text)
    
    return chunks

# Example text
text = """Artificial Intelligence (AI) simulates human intelligence in machines for tasks like visual perception, speech recognition, and language translation. It has evolved from rule-based systems to data-driven models, enhancing performance through machine learning and deep learning."""

# Define chunk size and overlap size
chunk_size = 80  # 80 characters per chunk
overlap_size = 10  # 10 characters overlap between chunks

# Get the chunks with overlap
chunks = split_text_with_overlap(text, chunk_size, overlap_size)

# Print the chunks and overlaps
for i in range(len(chunks)):
    print(f"Chunk {i+1}:")
    print(chunks[i])  # Print the chunk itself
    
    # If there's a next chunk, print the overlap between current and next chunk
    if i < len(chunks) - 1:
        overlap = chunks[i][-overlap_size:]  # Get the overlap part
        print(f"Overlap with Chunk {i+2}:")
        print(overlap)
    
    print("\n" + "="*50 + "\n")

from langchain.text_splitter import RecursiveCharacterTextSplitter

# Function to split text into chunks using recursive chunking
def split_text_recursive(text, chunk_size=80):
    # Initialize the RecursiveCharacterTextSplitter
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=chunk_size,  # Maximum size of each chunk (80 words)
        chunk_overlap=0         # No overlap between chunks
    )
    
    # Split the text into chunks
    chunks = text_splitter.split_text(text)
    
    return chunks

# Example text
text = """Artificial Intelligence (AI) simulates human intelligence in machines for tasks like visual perception, speech recognition, and language translation. It has evolved from rule-based systems to data-driven models, enhancing performance through machine learning and deep learning."""

# Split the text using recursive chunking
chunks = split_text_recursive(text, chunk_size=80)

# Print the resulting chunks
for i, chunk in enumerate(chunks):
    print(f"Chunk {i+1}:")
    print(chunk)
    print("="*50)

import os
from langchain_experimental.text_splitter import SemanticChunker
from langchain_openai.embeddings import OpenAIEmbeddings

# Set the OpenAI API key as an environment variable (Replace with your actual API key)
os.environ["OPENAI_API_KEY"] = "replace with your actual OpenAI API key" 

# Function to split text into semantic chunks
def split_text_semantically(text, breakpoint_type="percentile"):
    # Initialize the SemanticChunker with OpenAI embeddings
    text_splitter = SemanticChunker(OpenAIEmbeddings(), breakpoint_threshold_type=breakpoint_type)
    
    # Create documents (chunks)
    docs = text_splitter.create_documents([text])
    
    # Return the list of chunks
    return [doc.page_content for doc in docs]

def main():
    # Example content (State of the Union address or your own text)
    document_content = """
 Artificial Intelligence (AI) simulates human intelligence in machines for tasks like visual perception, speech recognition, and language translation. It has evolved from rule-based systems to data-driven models, enhancing performance through machine learning and deep learning.

Prompt Engineering involves designing input prompts to guide AI models in producing accurate and relevant responses, improving tasks such as text generation and summarization.
    """
    
    # Split text using the chosen threshold type (percentile)
    threshold_type = "percentile"
    print(f"\nChunks using {threshold_type} threshold:")
    chunks = split_text_semantically(document_content, breakpoint_type=threshold_type)
    
    # Print each chunk's content
    for idx, chunk in enumerate(chunks):
        print(f"Chunk {idx + 1}:\n{chunk}\n")
        
if __name__ == "__main__":
    main()

from langchain.chat_models import ChatOpenAI
from langchain.prompts import PromptTemplate
from langchain.chains import LLMChain
from langchain.agents import initialize_agent, Tool, AgentType

# Initialize OpenAI chat model (replace with your API key)
llm = ChatOpenAI(model="gpt-3.5-turbo", api_key="replace with your actual OpenAI API key")

# Step 1: Define Chunking and Summarization Prompt Template
chunk_prompt_template = """
You are given a large piece of text. Your job is to break it into smaller parts (chunks) if necessary and summarize each chunk.
Once all parts are summarized, combine them into a final summary. 
If the text is already small enough to process at once, provide a full summary in one step. 
Please summarize the following text:\n{input}
"""
chunk_prompt = PromptTemplate(input_variables=["input"], template=chunk_prompt_template)

# Step 2: Define Chunk Processing Tool
def chunk_processing_tool(query):
    """Processes text chunks and generates summaries using the defined prompt."""
    chunk_chain = LLMChain(llm=llm, prompt=chunk_prompt)
    print(f"Processing chunk:\n{query}\n")  # Show the chunk being processed
    return chunk_chain.run(input=query)

# Step 3: Define External Tool (Optional, can be used to fetch extra information if needed)
def external_tool(query):
    """Simulates an external tool that could fetch additional information."""
    return f"External response based on the query: {query}"

# Step 4: Initialize the agent with tools
tools = [
    Tool(
        name="Chunk Processing",
        func=chunk_processing_tool,
        description="Processes text chunks and generates summaries."
    ),
    Tool(
        name="External Query",
        func=external_tool,
        description="Fetches additional data to enhance chunk processing."
    )
]

# Initialize the agent with defined tools and zero-shot capabilities
agent = initialize_agent(
    tools=tools,
    agent_type=AgentType.ZERO_SHOT_REACT_DESCRIPTION,
    llm=llm,
    verbose=True
)

# Step 5: Agentic Chunk Processing Function
def agent_process_chunks(text):
    """Uses the agent to process text chunks and generate a final output."""
    # Step 1: Chunking the text into smaller, manageable sections
    def chunk_text(text, chunk_size=500):
        """Splits large text into smaller chunks."""
        return [text[i:i + chunk_size] for i in range(0, len(text), chunk_size)]

    chunks = chunk_text(text)
    
    # Step 2: Process each chunk with the agent
    chunk_results = []
    for idx, chunk in enumerate(chunks):
        print(f"Processing chunk {idx + 1}/{len(chunks)}...")
        response = agent.invoke({"input": chunk})  # Process chunk using the agent
        chunk_results.append(response['output'])  # Collect the chunk result

    # Step 3: Combine the chunk results into a final output
    final_output = "\n".join(chunk_results)
    return final_output

# Step 6: Running the agent on an example large text input
if __name__ == "__main__":
    # Example large text content
    text_to_process = """
    Artificial intelligence (AI) is transforming industries by enabling machines to perform tasks that
    previously required human intelligence. From healthcare to finance, AI is driving innovation and improving
    efficiency. For instance, in healthcare, AI algorithms assist doctors in diagnosing diseases, interpreting
    medical images, and predicting patient outcomes. Meanwhile, in finance, AI helps detect fraud, manage
    investments, and automate customer service.

    However, the widespread adoption of AI also raises ethical concerns. Issues like privacy invasion,
    algorithmic bias, and the potential loss of jobs due to automation are significant challenges. Experts
    argue that it's essential to develop AI responsibly to ensure that it benefits society as a whole.
    Proper regulations, transparency, and accountability can help address these issues, ensuring that AI
    technologies are used for the greater good.

    Beyond individual industries, AI is also impacting the global economy. Nations are investing heavily
    in AI research and development to maintain a competitive edge. This technological race could redefine
    global power dynamics, with countries that excel in AI leading the way in economic and military strength.
    Despite the potential for AI to contribute positively to society, its development and application require
    careful consideration of ethical, legal, and societal implications.
    """

    # Process the text and print the final result
    final_result = agent_process_chunks(text_to_process)
    print("\nFinal Output:\n", final_result)