From b4f6066a572d6aeef4a1c6e563b0e40235f363c9 Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Wed, 28 Feb 2024 10:04:15 -0800 Subject: [PATCH] docs: airbyte github cookbook (#18275) --- cookbook/airbyte_github.ipynb | 200 ++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 cookbook/airbyte_github.ipynb diff --git a/cookbook/airbyte_github.ipynb b/cookbook/airbyte_github.ipynb new file mode 100644 index 0000000000..349c235be6 --- /dev/null +++ b/cookbook/airbyte_github.ipynb @@ -0,0 +1,200 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -qU langchain-airbyte" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "\n", + "GITHUB_TOKEN = getpass.getpass()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_airbyte import AirbyteLoader\n", + "from langchain_core.prompts import PromptTemplate\n", + "\n", + "loader = AirbyteLoader(\n", + " source=\"source-github\",\n", + " stream=\"pull_requests\",\n", + " config={\n", + " \"credentials\": {\"personal_access_token\": GITHUB_TOKEN},\n", + " \"repositories\": [\"langchain-ai/langchain\"],\n", + " },\n", + " template=PromptTemplate.from_template(\n", + " \"\"\"# {title}\n", + "by {user[login]}\n", + "\n", + "{body}\"\"\"\n", + " ),\n", + " include_metadata=False,\n", + ")\n", + "docs = loader.load()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Updated partners/ibm README\n", + "by williamdevena\n", + "\n", + "## PR title\n", + "partners: changed the README file for the IBM Watson AI integration in the libs/partners/ibm folder.\n", + "\n", + "## PR message\n", + "Description: Changed the README file of partners/ibm following the docs on https://python.langchain.com/docs/integrations/llms/ibm_watsonx\n", + "\n", + "The README includes:\n", + "\n", + "- Brief description\n", + "- Installation\n", + "- Setting-up instructions (API key, project id, ...)\n", + "- Basic usage:\n", + " - Loading the model\n", + " - Direct inference\n", + " - Chain invoking\n", + " - Streaming the model output\n", + " \n", + "Issue: https://github.com/langchain-ai/langchain/issues/17545\n", + "\n", + "Dependencies: None\n", + "\n", + "Twitter handle: None\n" + ] + } + ], + "source": [ + "print(docs[-2].page_content)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "10283" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(docs)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "import tiktoken\n", + "from langchain_community.vectorstores import Chroma\n", + "from langchain_openai import OpenAIEmbeddings\n", + "\n", + "enc = tiktoken.get_encoding(\"cl100k_base\")\n", + "\n", + "vectorstore = Chroma.from_documents(\n", + " docs,\n", + " embedding=OpenAIEmbeddings(\n", + " disallowed_special=(enc.special_tokens_set - {\"<|endofprompt|>\"})\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "retriever = vectorstore.as_retriever()" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(page_content='# Updated partners/ibm README\\nby williamdevena\\n\\n## PR title\\r\\npartners: changed the README file for the IBM Watson AI integration in the libs/partners/ibm folder.\\r\\n\\r\\n## PR message\\r\\nDescription: Changed the README file of partners/ibm following the docs on https://python.langchain.com/docs/integrations/llms/ibm_watsonx\\r\\n\\r\\nThe README includes:\\r\\n\\r\\n- Brief description\\r\\n- Installation\\r\\n- Setting-up instructions (API key, project id, ...)\\r\\n- Basic usage:\\r\\n - Loading the model\\r\\n - Direct inference\\r\\n - Chain invoking\\r\\n - Streaming the model output\\r\\n \\r\\nIssue: https://github.com/langchain-ai/langchain/issues/17545\\r\\n\\r\\nDependencies: None\\r\\n\\r\\nTwitter handle: None'),\n", + " Document(page_content='# Updated partners/ibm README\\nby williamdevena\\n\\n## PR title\\r\\npartners: changed the README file for the IBM Watson AI integration in the `libs/partners/ibm` folder. \\r\\n\\r\\n\\r\\n\\r\\n## PR message\\r\\n- **Description:** Changed the README file of partners/ibm following the docs on https://python.langchain.com/docs/integrations/llms/ibm_watsonx\\r\\n\\r\\n The README includes:\\r\\n - Brief description\\r\\n - Installation\\r\\n - Setting-up instructions (API key, project id, ...)\\r\\n - Basic usage:\\r\\n - Loading the model\\r\\n - Direct inference\\r\\n - Chain invoking\\r\\n - Streaming the model output\\r\\n\\r\\n\\r\\n- **Issue:** #17545\\r\\n- **Dependencies:** None\\r\\n- **Twitter handle:** None'),\n", + " Document(page_content='# IBM: added partners package `langchain_ibm`, added llm\\nby MateuszOssGit\\n\\n - **Description:** Added `langchain_ibm` as an langchain partners package of IBM [watsonx.ai](https://www.ibm.com/products/watsonx-ai) LLM provider (`WatsonxLLM`)\\r\\n - **Dependencies:** [ibm-watsonx-ai](https://pypi.org/project/ibm-watsonx-ai/),\\r\\n - **Tag maintainer:** : \\r\\n\\r\\nPlease make sure your PR is passing linting and testing before submitting. Run `make format`, `make lint` and `make test` to check this locally. ✅'),\n", + " Document(page_content='# Add WatsonX support\\nby baptistebignaud\\n\\nIt is a connector to use a LLM from WatsonX.\\r\\nIt requires python SDK \"ibm-generative-ai\"\\r\\n\\r\\n(It might not be perfect since it is my first PR on a public repository 😄)')]" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "retriever.invoke(\"pull requests related to IBM\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}