Move web Dockerfile to single stage since multi-stage has been causing issues w/ github actions

Co-authored-by: Matthieu Boret <matthieu.boret@fr.clara.net>

.github/workflows/docker-build-push-backend-container-on-tag.yml

@@ -0,0 +1,44 @@
+name: Build and Push Backend Image on Tag
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_TOKEN }}
+
+    - name: Backend Image Docker Build and Push
+      uses: docker/build-push-action@v2
+      with:
+        context: ./backend
+        file: ./backend/Dockerfile
+        platforms: linux/amd64,linux/arm64
+        push: true
+        tags: |
+          danswer/danswer-backend:${{ github.ref_name }}
+          danswer/danswer-backend:latest
+        build-args: |
+          DANSWER_VERSION=${{ github.ref_name }}
+
+    - name: Run Trivy vulnerability scanner
+      uses: aquasecurity/trivy-action@master
+      with:
+        # To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend
+        image-ref: docker.io/danswer/danswer-backend:${{ github.ref_name }}
+        severity: 'CRITICAL,HIGH'
+        trivyignores: ./backend/.trivyignore

.github/workflows/docker-build-push-model-server-container-on-tag.yml

@@ -0,0 +1,42 @@
+name: Build and Push Model Server Image on Tag
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_TOKEN }}
+
+    - name: Model Server Image Docker Build and Push
+      uses: docker/build-push-action@v2
+      with:
+        context: ./backend
+        file: ./backend/Dockerfile.model_server
+        platforms: linux/amd64,linux/arm64
+        push: true
+        tags: |
+          danswer/danswer-model-server:${{ github.ref_name }}
+          danswer/danswer-model-server:latest
+        build-args: |
+          DANSWER_VERSION=${{ github.ref_name }}
+
+    - name: Run Trivy vulnerability scanner
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: docker.io/danswer/danswer-model-server:${{ github.ref_name }}
+        severity: 'CRITICAL,HIGH'

.github/workflows/docker-build-push-web-container-on-tag.yml

@@ -0,0 +1,42 @@
+name: Build and Push Web Image on Tag
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_TOKEN }}
+
+    - name: Web Image Docker Build and Push
+      uses: docker/build-push-action@v2
+      with:
+        context: ./web
+        file: ./web/Dockerfile
+        platforms: linux/amd64,linux/arm64
+        push: true
+        tags: |
+          danswer/danswer-web-server:${{ github.ref_name }}
+          danswer/danswer-web-server:latest
+        build-args: |
+          DANSWER_VERSION=${{ github.ref_name }}
+
+    - name: Run Trivy vulnerability scanner
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: docker.io/danswer/danswer-web-server:${{ github.ref_name }}
+        severity: 'CRITICAL,HIGH'

.github/workflows/docker-tag-latest.yml

@@ -0,0 +1,32 @@
+name: Tag Latest Version
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'The version (ie v0.0.1) to tag as latest'
+        required: true
+
+jobs:
+  tag:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_TOKEN }}
+
+    - name: Enable Docker CLI experimental features
+      run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV
+
+    - name: Pull, Tag and Push Web Server Image
+      run: |
+        docker buildx imagetools create -t danswer/danswer-web-server:latest danswer/danswer-web-server:${{ github.event.inputs.version }}
+
+    - name: Pull, Tag and Push API Server Image
+      run: |
+        docker buildx imagetools create -t danswer/danswer-backend:latest danswer/danswer-backend:${{ github.event.inputs.version }}

.github/workflows/pr-python-checks.yml

@@ -0,0 +1,48 @@
+name: Python Checks
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  mypy-check:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: |
+          backend/requirements/default.txt
+          backend/requirements/dev.txt
+          backend/requirements/model_server.txt
+    - run: |
+        python -m pip install --upgrade pip
+        pip install -r backend/requirements/default.txt
+        pip install -r backend/requirements/dev.txt
+        pip install -r backend/requirements/model_server.txt
+
+    - name: Run MyPy
+      run: |
+        cd backend
+        mypy .
+
+    - name: Run ruff
+      run: |
+        cd backend
+        ruff .
+
+    - name: Check import order with reorder-python-imports
+      run: |
+        cd backend
+        find ./danswer -name "*.py" | xargs reorder-python-imports --py311-plus
+
+    - name: Check code formatting with Black
+      run: |
+        cd backend
+        black --check .

.github/workflows/pr-python-tests.yml

@@ -0,0 +1,35 @@
+name: Python Unit Tests
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  backend-check:
+    runs-on: ubuntu-latest
+
+    env:
+      PYTHONPATH: ./backend
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: |
+          backend/requirements/default.txt
+          backend/requirements/dev.txt
+
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r backend/requirements/default.txt
+        pip install -r backend/requirements/dev.txt
+
+    - name: Run Tests
+      shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
+      run: py.test -o junit_family=xunit2 -xv --ff backend/tests/unit

.github/workflows/pr-quality-checks.yml

@@ -0,0 +1,21 @@
+name: Quality Checks PR
+concurrency:
+  group: Quality-Checks-PR-${{ github.head_ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request: null
+
+jobs:
+  quality-checks:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - uses: pre-commit/action@v3.0.0
+      with:
+        extra_args: --from-ref ${{ github.event.pull_request.base.sha }} --to-ref ${{ github.event.pull_request.head.sha }}

.gitignore

@@ -0,0 +1,7 @@
+.env
+.DS_store
+.venv
+.mypy_cache
+.idea
+/deployment/data/nginx/app.conf
+.vscode/launch.json

.pre-commit-config.yaml

@@ -0,0 +1,65 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
+    - id: black
+      language_version: python3.11
+
+  - repo: https://github.com/asottile/reorder_python_imports
+    rev: v3.9.0
+    hooks:
+    - id: reorder-python-imports
+      args: ['--py311-plus', '--application-directories=backend/']
+      # need to ignore alembic files, since reorder-python-imports gets confused
+      # and thinks that alembic is a local package since there is a folder
+      # in the backend directory called `alembic`
+      exclude: ^backend/alembic/
+
+  # These settings will remove unused imports with side effects
+  # Note: The repo currently does not and should not have imports with side effects
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.0
+    hooks:
+      - id: autoflake
+        args: [ '--remove-all-unused-imports', '--remove-unused-variables', '--in-place' , '--recursive']
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.0.286
+    hooks:
+      - id: ruff
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.1.0
+    hooks:
+    - id: prettier
+      types_or: [html, css, javascript, ts, tsx]
+      additional_dependencies:
+      - prettier
+
+  # We would like to have a mypy pre-commit hook, but due to the fact that
+  # pre-commit runs in it's own isolated environment, we would need to install
+  # and keep in sync all dependencies so mypy has access to the appropriate type
+  # stubs. This does not seem worth it at the moment, so for now we will stick to
+  # having mypy run via Github Actions / manually by contributors
+  # - repo: https://github.com/pre-commit/mirrors-mypy
+  #   rev: v1.1.1
+  #   hooks:
+  #     - id: mypy
+  #       exclude: ^tests/
+  #       # below are needed for type stubs since pre-commit runs in it's own
+  #       # isolated environment. Unfortunately, this needs to be kept in sync
+  #       # with requirements/dev.txt + requirements/default.txt
+  #       additional_dependencies: [
+  #         alembic==1.10.4,
+  #         types-beautifulsoup4==4.12.0.3,
+  #         types-html5lib==1.1.11.13,
+  #         types-oauthlib==3.2.0.9,
+  #         types-psycopg2==2.9.21.10,
+  #         types-python-dateutil==2.8.19.13,
+  #         types-regex==2023.3.23.1,
+  #         types-requests==2.28.11.17,
+  #         types-retry==0.9.9.3,
+  #         types-urllib3==1.26.25.11
+  #       ]
+  #       # TODO: add back once errors are addressed
+  #       # args: [--strict]

.vscode/launch.template.jsonc

@@ -0,0 +1,105 @@
+/*
+
+  Copy this file into '.vscode/launch.json' or merge its
+  contents into your existing configurations.
+
+*/
+
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Web Server",
+            "type": "node",
+            "request": "launch",
+            "cwd": "${workspaceRoot}/web",
+            "runtimeExecutable": "npm",
+            "runtimeArgs": [
+                "run", "dev"
+            ],
+            "console": "integratedTerminal"
+        },
+        {
+            "name": "Model Server",
+            "type": "python",
+            "request": "launch",
+            "module": "uvicorn",
+            "cwd": "${workspaceFolder}/backend",
+            "env": {
+                "LOG_LEVEL": "DEBUG",
+                "PYTHONUNBUFFERED": "1"
+            },
+            "args": [
+                "model_server.main:app",
+                "--reload",
+                "--port",
+                "9000"
+            ]
+        },
+        {
+            "name": "API Server",
+            "type": "python",
+            "request": "launch",
+            "module": "uvicorn",
+            "cwd": "${workspaceFolder}/backend",
+            "env": {
+                "LOG_ALL_MODEL_INTERACTIONS": "True",
+                "LOG_LEVEL": "DEBUG",
+                "PYTHONUNBUFFERED": "1"
+            },
+            "args": [
+                "danswer.main:app",
+                "--reload",
+                "--port",
+                "8080"
+            ]
+        },
+        {
+            "name": "Indexing",
+            "type": "python",
+            "request": "launch",
+            "program": "danswer/background/update.py",
+            "cwd": "${workspaceFolder}/backend",
+            "env": {
+                "ENABLE_MINI_CHUNK": "false",
+                "LOG_LEVEL": "DEBUG",
+                "PYTHONUNBUFFERED": "1",
+                "PYTHONPATH": "."
+            }
+        },
+        // Celery and all async jobs, usually would include indexing as well but this is handled separately above for dev
+        {
+            "name": "Background Jobs",
+            "type": "python",
+            "request": "launch",
+            "program": "scripts/dev_run_background_jobs.py",
+            "cwd": "${workspaceFolder}/backend",
+            "env": {
+                "LOG_LEVEL": "DEBUG",
+                "PYTHONUNBUFFERED": "1",
+                "PYTHONPATH": "."
+            },
+            "args": [
+                "--no-indexing"
+            ]
+        },
+        // For the listner to access the Slack API,
+        // DANSWER_BOT_SLACK_APP_TOKEN & DANSWER_BOT_SLACK_BOT_TOKEN need to be set in .env file located in the root of the project
+        {
+            "name": "Slack Bot",
+            "type": "python",
+            "request": "launch",
+            "program": "danswer/danswerbot/slack/listener.py",
+            "cwd": "${workspaceFolder}/backend",
+            "envFile": "${workspaceFolder}/.env",
+            "env": {
+                "LOG_LEVEL": "DEBUG",
+                "PYTHONUNBUFFERED": "1",
+                "PYTHONPATH": "."
+            }
+        }
+    ]
+}

CONTRIBUTING.md

@@ -0,0 +1,187 @@
+<!-- DANSWER_METADATA={"link": "https://github.com/danswer-ai/danswer/blob/main/CONTRIBUTING.md"} -->
+
+# Contributing to Danswer
+Hey there! We are so excited that you're interested in Danswer.
+
+As an open source project in a rapidly changing space, we welcome all contributions.
+
+
+## 💃 Guidelines
+### Contribution Opportunities
+The [GitHub Issues](https://github.com/danswer-ai/danswer/issues) page is a great place to start for contribution ideas.
+
+Issues that have been explicitly approved by the maintainers (aligned with the direction of the project)
+will be marked with the `approved by maintainers` label.
+Issues marked `good first issue` are an especially great place to start.
+
+**Connectors** to other tools are another great place to contribute. For details on how, refer to this
+[README.md](https://github.com/danswer-ai/danswer/blob/main/backend/danswer/connectors/README.md).
+
+If you have a new/different contribution in mind, we'd love to hear about it!
+Your input is vital to making sure that Danswer moves in the right direction.
+Before starting on implementation, please raise a GitHub issue.
+
+And always feel free to message us (Chris Weaver / Yuhong Sun) on 
+[Slack](https://join.slack.com/t/danswer/shared_invite/zt-2afut44lv-Rw3kSWu6_OmdAXRpCv80DQ) / 
+[Discord](https://discord.gg/TDJ59cGV2X) directly about anything at all. 
+
+
+### Contributing Code
+To contribute to this project, please follow the
+["fork and pull request"](https://docs.github.com/en/get-started/quickstart/contributing-to-projects) workflow.
+When opening a pull request, mention related issues and feel free to tag relevant maintainers.
+
+Before creating a pull request please make sure that the new changes conform to the formatting and linting requirements.
+See the [Formatting and Linting](#-formatting-and-linting) section for how to run these checks locally.
+
+
+### Getting Help 🙋
+Our goal is to make contributing as easy as possible. If you run into any issues please don't hesitate to reach out.
+That way we can help future contributors and users can avoid the same issue.
+
+We also have support channels and generally interesting discussions on our
+[Slack](https://join.slack.com/t/danswer/shared_invite/zt-2afut44lv-Rw3kSWu6_OmdAXRpCv80DQ)
+and 
+[Discord](https://discord.gg/TDJ59cGV2X).
+
+We would love to see you there!
+
+
+## Get Started 🚀
+Danswer being a fully functional app, relies on some external pieces of software, specifically:
+- [Postgres](https://www.postgresql.org/) (Relational DB)
+- [Vespa](https://vespa.ai/) (Vector DB/Search Engine)
+
+This guide provides instructions to set up the Danswer specific services outside of Docker because it's easier for
+development purposes but also feel free to just use the containers and update with local changes by providing the
+`--build` flag.
+
+
+### Local Set Up
+It is recommended to use Python version 3.11
+
+If using a lower version, modifications will have to be made to the code.
+If using a higher version, the version of Tensorflow we use may not be available for your platform.
+
+
+#### Installing Requirements
+Currently, we use pip and recommend creating a virtual environment.
+
+For convenience here's a command for it:
+```bash
+python -m venv .venv
+source .venv/bin/activate
+```
+_For Windows, activate the virtual environment using Command Prompt:_
+```bash
+.venv\Scripts\activate
+```
+If using PowerShell, the command slightly differs:
+```powershell
+.venv\Scripts\Activate.ps1
+```
+
+Install the required python dependencies:
+```bash
+pip install -r danswer/backend/requirements/default.txt
+pip install -r danswer/backend/requirements/dev.txt
+pip install -r danswer/backend/requirements/model_server.txt
+```
+
+Install [Node.js and npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) for the frontend.
+Once the above is done, navigate to `danswer/web` run:
+```bash
+npm i
+```
+
+Install Playwright (required by the Web Connector)
+
+> Note: If you have just done the pip install, open a new terminal and source the python virtual-env again.
+This will update the path to include playwright
+
+Then install Playwright by running:
+```bash
+playwright install
+```
+
+
+#### Dependent Docker Containers
+First navigate to `danswer/deployment/docker_compose`, then start up Vespa and Postgres with:
+```bash
+docker compose -f docker-compose.dev.yml -p danswer-stack up -d index relational_db
+```
+(index refers to Vespa and relational_db refers to Postgres)
+
+#### Running Danswer
+To start the frontend, navigate to `danswer/web` and run:
+```bash
+npm run dev
+```
+
+Next, start the model server which runs the local NLP models.
+Navigate to `danswer/backend` and run:
+```bash
+uvicorn model_server.main:app --reload --port 9000
+```
+_For Windows (for compatibility with both PowerShell and Command Prompt):_
+```bash
+powershell -Command "
+    uvicorn model_server.main:app --reload --port 9000
+"
+```
+
+The first time running Danswer, you will need to run the DB migrations for Postgres.
+After the first time, this is no longer required unless the DB models change.
+
+Navigate to `danswer/backend` and with the venv active, run:
+```bash
+alembic upgrade head
+```
+
+Next, start the task queue which orchestrates the background jobs.
+Jobs that take more time are run async from the API server.
+
+Still in `danswer/backend`, run:
+```bash
+python ./scripts/dev_run_background_jobs.py
+```
+
+To run the backend API server, navigate back to `danswer/backend` and run:
+```bash
+AUTH_TYPE=disabled uvicorn danswer.main:app --reload --port 8080
+```
+_For Windows (for compatibility with both PowerShell and Command Prompt):_
+```bash
+powershell -Command "
+    $env:AUTH_TYPE='disabled'
+    uvicorn danswer.main:app --reload --port 8080 
+"
+```
+
+Note: if you need finer logging, add the additional environment variable `LOG_LEVEL=DEBUG` to the relevant services.
+
+### Formatting and Linting
+#### Backend
+For the backend, you'll need to setup pre-commit hooks (black / reorder-python-imports).
+First, install pre-commit (if you don't have it already) following the instructions
+[here](https://pre-commit.com/#installation).
+Then, from the `danswer/backend` directory, run:
+```bash
+pre-commit install
+```
+
+Additionally, we use `mypy` for static type checking.
+Danswer is fully type-annotated, and we would like to keep it that way! 
+To run the mypy checks manually, run `python -m mypy .` from the `danswer/backend` directory.
+
+
+#### Web
+We use `prettier` for formatting. The desired version (2.8.8) will be installed via a `npm i` from the `danswer/web` directory. 
+To run the formatter, use `npx prettier --write .` from the `danswer/web` directory.
+Please double check that prettier passes before creating a pull request.
+
+
+### Release Process
+Danswer follows the semver versioning standard.
+A set of Docker containers will be pushed automatically to DockerHub with every tag.
+You can see the containers [here](https://hub.docker.com/search?q=danswer%2F).

LICENSE

@@ -1,661 +1,21 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
-  A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate.  Many developers of free software are heartened and
-encouraged by the resulting cooperation.  However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
-  The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community.  It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
-  An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals.  This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU Affero General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Remote Network Interaction; Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published
-    by the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-<https://www.gnu.org/licenses/>.
+MIT License
+
+Copyright (c) 2023 Yuhong Sun, Chris Weaver
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,109 @@
+<!-- DANSWER_METADATA={"link": "https://github.com/danswer-ai/danswer/blob/main/README.md"} -->
+
+<h2 align="center">
+<a href="https://www.danswer.ai/"> <img width="50%" src="https://github.com/danswer-owners/danswer/blob/1fabd9372d66cd54238847197c33f091a724803b/DanswerWithName.png?raw=true)" /></a>
+</h2>
+
+<p align="center">
+<p align="center">Open Source Gen-AI Chat + Unified Search.</p>
+
+<p align="center">
+<a href="https://docs.danswer.dev/" target="_blank">
+    <img src="https://img.shields.io/badge/docs-view-blue" alt="Documentation">
+</a>
+<a href="https://join.slack.com/t/danswer/shared_invite/zt-2afut44lv-Rw3kSWu6_OmdAXRpCv80DQ" target="_blank">
+    <img src="https://img.shields.io/badge/slack-join-blue.svg?logo=slack" alt="Slack">
+</a>
+<a href="https://discord.gg/TDJ59cGV2X" target="_blank">
+    <img src="https://img.shields.io/badge/discord-join-blue.svg?logo=discord&logoColor=white" alt="Discord">
+</a>
+<a href="https://github.com/danswer-ai/danswer/blob/main/README.md" target="_blank">
+    <img src="https://img.shields.io/static/v1?label=license&message=MIT&color=blue" alt="License">
+</a>
+</p>
+
+<strong>[Danswer](https://www.danswer.ai/)</strong> is the AI Assistant connected to your company's docs, apps, and people. 
+Danswer provides a Chat interface and plugs into any LLM of your choice. Danswer can be deployed anywhere and for any 
+scale - on a laptop, on-premise, or to cloud. Since you own the deployment, your user data and chats are fully in your 
+own control. Danswer is MIT licensed and designed to be modular and easily extensible. The system also comes fully ready 
+for production usage with user authentication, role management (admin/basic users), chat persistence, and a UI for 
+configuring Personas (AI Assistants) and their Prompts.
+
+Danswer also serves as a Unified Search across all common workplace tools such as Slack, Google Drive, Confluence, etc.
+By combining LLMs and team specific knowledge, Danswer becomes a subject matter expert for the team. Imagine ChatGPT if
+it had access to your team's unique knowledge! It enables questions such as "A customer wants feature X, is this already
+supported?" or "Where's the pull request for feature Y?"
+
+<h3>Usage</h3>
+
+Danswer Web App:
+
+https://github.com/danswer-ai/danswer/assets/32520769/563be14c-9304-47b5-bf0a-9049c2b6f410
+
+
+Or, plug Danswer into your existing Slack workflows (more integrations to come 😁):
+
+https://github.com/danswer-ai/danswer/assets/25087905/3e19739b-d178-4371-9a38-011430bdec1b
+
+
+For more details on the Admin UI to manage connectors and users, check out our 
+<strong><a href="https://www.youtube.com/watch?v=geNzY1nbCnU">Full Video Demo</a></strong>!
+
+## Deployment
+
+Danswer can easily be run locally (even on a laptop) or deployed on a virtual machine with a single
+`docker compose` command. Checkout our [docs](https://docs.danswer.dev/quickstart) to learn more.
+
+We also have built-in support for deployment on Kubernetes. Files for that can be found [here](https://github.com/danswer-ai/danswer/tree/main/deployment/kubernetes).
+
+
+## 💃 Main Features 
+* Chat UI with the ability to select documents to chat with.
+* Create custom AI Assistants with different prompts and backing knowledge sets.
+* Connect Danswer with LLM of your choice (self-host for a fully airgapped solution).
+* Document Search + AI Answers for natural language queries.
+* Connectors to all common workplace tools like Google Drive, Confluence, Slack, etc.
+* Slack integration to get answers and search results directly in Slack.
+
+
+## 🚧 Roadmap
+* Chat/Prompt sharing with specific teammates and user groups.
+* Multi-Model model support, chat with images, video etc.
+* Choosing between LLMs and parameters during chat session.
+* Tool calling and agent configurations options.
+* Organizational understanding and ability to locate and suggest experts from your team.
+
+
+## Other Noteable Benefits of Danswer
+* User Authentication with document level access management.
+* Best in class Hybrid Search across all sources (BM-25 + prefix aware embedding models).
+* Admin Dashboard to configure connectors, document-sets, access, etc.
+* Custom deep learning models + learn from user feedback.
+* Easy deployment and ability to host Danswer anywhere of your choosing.
+
+
+## 🔌 Connectors
+Efficiently pulls the latest changes from:
+  * Slack
+  * GitHub
+  * Google Drive
+  * Confluence
+  * Jira
+  * Zendesk
+  * Gmail
+  * Notion
+  * Gong
+  * Slab
+  * Linear
+  * Productboard
+  * Guru
+  * Bookstack
+  * Document360
+  * Sharepoint
+  * Hubspot
+  * Local Files
+  * Websites
+  * And more ...
+
+## 💡 Contributing
+Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md) for more details.

backend/.dockerignore

@@ -0,0 +1,17 @@
+**/__pycache__
+venv/
+env/
+*.egg-info
+.cache
+.git/
+.svn/
+.vscode/
+.idea/
+*.log
+log/
+.env
+secrets.yaml
+build/
+dist/
+.coverage
+htmlcov/

backend/.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+.mypy_cache
+.idea/
+site_crawls/
+.ipynb_checkpoints/
+api_keys.py
+*ipynb
+.env
+vespa-app.zip
+dynamic_config_storage/
+celerybeat-schedule*

backend/.trivyignore

@@ -0,0 +1,46 @@
+# https://github.com/madler/zlib/issues/868
+# Pulled in with base Debian image, it's part of the contrib folder but unused
+# zlib1g is fine
+# Will be gone with Debian image upgrade
+# No impact in our settings
+CVE-2023-45853
+
+# krb5 related, worst case is denial of service by resource exhaustion
+# Accept the risk
+CVE-2024-26458
+CVE-2024-26461
+CVE-2024-26462
+CVE-2024-26458
+CVE-2024-26461
+CVE-2024-26462
+CVE-2024-26458
+CVE-2024-26461
+CVE-2024-26462
+CVE-2024-26458
+CVE-2024-26461
+CVE-2024-26462
+
+# Specific to Firefox which we do not use
+# No impact in our settings
+CVE-2024-0743
+
+# bind9 related, worst case is denial of service by CPU resource exhaustion
+# Accept the risk
+CVE-2023-50387
+CVE-2023-50868
+CVE-2023-50387
+CVE-2023-50868
+
+# libexpat1, XML parsing resource exhaustion
+# We don't parse any user provided XMLs
+# No impact in our settings
+CVE-2023-52425
+CVE-2024-28757
+
+# sqlite, only used by NLTK library to grab word lemmatizer and stopwords
+# No impact in our settings
+CVE-2023-7104
+
+# libharfbuzz0b, O(n^2) growth, worst case is denial of service
+# Accept the risk
+CVE-2023-25193

backend/Dockerfile

@@ -0,0 +1,66 @@
+FROM python:3.11.7-slim-bookworm
+
+LABEL com.danswer.maintainer="founders@danswer.ai"
+LABEL com.danswer.description="This image is for the backend of Danswer. It is MIT Licensed and \
+free for all to use. You can find it at https://hub.docker.com/r/danswer/danswer-backend. For \
+more details, visit https://github.com/danswer-ai/danswer."
+
+# Default DANSWER_VERSION, typically overriden during builds by GitHub Actions.
+ARG DANSWER_VERSION=0.3-dev
+ENV DANSWER_VERSION=${DANSWER_VERSION}
+RUN echo "DANSWER_VERSION: ${DANSWER_VERSION}"
+
+# Install system dependencies
+# cmake needed for psycopg (postgres)
+# libpq-dev needed for psycopg (postgres)
+# curl included just for users' convenience
+# zip for Vespa step futher down
+# ca-certificates for HTTPS
+RUN apt-get update && \
+    apt-get install -y cmake curl zip ca-certificates libgnutls30=3.7.9-2+deb12u2 \
+    libblkid1=2.38.1-5+deb12u1 libmount1=2.38.1-5+deb12u1 libsmartcols1=2.38.1-5+deb12u1 \
+    libuuid1=2.38.1-5+deb12u1 && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get clean
+
+# Install Python dependencies
+# Remove py which is pulled in by retry, py is not needed and is a CVE
+COPY ./requirements/default.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt && \
+    pip uninstall -y py && \
+    playwright install chromium && playwright install-deps chromium && \
+    ln -s /usr/local/bin/supervisord /usr/bin/supervisord
+
+# Cleanup for CVEs and size reduction
+# https://github.com/tornadoweb/tornado/issues/3107
+# xserver-common and xvfb included by playwright installation but not needed after
+# perl-base is part of the base Python Debian image but not needed for Danswer functionality
+# perl-base could only be removed with --allow-remove-essential
+RUN apt-get remove -y --allow-remove-essential perl-base xserver-common xvfb cmake \
+    libldap-2.5-0 libldap-2.5-0 && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm /usr/local/lib/python3.11/site-packages/tornado/test/test.key
+
+# Pre-downloading models for setups with limited egress
+RUN python -c "from transformers import AutoTokenizer; AutoTokenizer.from_pretrained('intfloat/e5-base-v2')"
+
+# Pre-downloading NLTK for setups with limited egress
+RUN python -c "import nltk; \
+nltk.download('stopwords', quiet=True); \
+nltk.download('wordnet', quiet=True); \
+nltk.download('punkt', quiet=True);"
+
+# Set up application files
+WORKDIR /app
+COPY ./danswer /app/danswer
+COPY ./shared_configs /app/shared_configs
+COPY ./alembic /app/alembic
+COPY ./alembic.ini /app/alembic.ini
+COPY supervisord.conf /usr/etc/supervisord.conf
+
+ENV PYTHONPATH /app
+
+# Default command which does nothing
+# This container is used by api server and background which specify their own CMD
+CMD ["tail", "-f", "/dev/null"]

backend/Dockerfile.model_server

@@ -0,0 +1,46 @@
+FROM python:3.11.7-slim-bookworm
+
+LABEL com.danswer.maintainer="founders@danswer.ai"
+LABEL com.danswer.description="This image is for the Danswer model server which runs all of the \
+AI models for Danswer. This container and all the code is MIT Licensed and free for all to use. \
+You can find it at https://hub.docker.com/r/danswer/danswer-model-server. For more details, \
+visit https://github.com/danswer-ai/danswer."
+
+# Default DANSWER_VERSION, typically overriden during builds by GitHub Actions.
+ARG DANSWER_VERSION=0.3-dev
+ENV DANSWER_VERSION=${DANSWER_VERSION}
+RUN echo "DANSWER_VERSION: ${DANSWER_VERSION}"
+
+COPY ./requirements/model_server.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt
+
+RUN apt-get remove -y --allow-remove-essential perl-base && \
+    apt-get autoremove -y
+
+# Pre-downloading models for setups with limited egress
+RUN python -c "from transformers import AutoModel, AutoTokenizer, TFDistilBertForSequenceClassification; \
+from huggingface_hub import snapshot_download; \
+AutoTokenizer.from_pretrained('danswer/intent-model'); \
+AutoTokenizer.from_pretrained('intfloat/e5-base-v2'); \
+AutoTokenizer.from_pretrained('mixedbread-ai/mxbai-rerank-xsmall-v1'); \
+snapshot_download('danswer/intent-model'); \
+snapshot_download('intfloat/e5-base-v2'); \
+snapshot_download('mixedbread-ai/mxbai-rerank-xsmall-v1')"
+
+WORKDIR /app
+
+# Utils used by model server
+COPY ./danswer/utils/logger.py /app/danswer/utils/logger.py
+
+# Place to fetch version information
+COPY ./danswer/__init__.py /app/danswer/__init__.py
+
+# Shared between Danswer Backend and Model Server
+COPY ./shared_configs /app/shared_configs
+
+# Model Server main code
+COPY ./model_server /app/model_server
+
+ENV PYTHONPATH /app
+
+CMD ["uvicorn", "model_server.main:app", "--host", "0.0.0.0", "--port", "9000"]

backend/alembic.ini

@@ -0,0 +1,108 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+hooks = black
+black.type = console_scripts
+black.entrypoint = black
+black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

backend/alembic/README.md

@@ -0,0 +1,19 @@
+<!-- DANSWER_METADATA={"link": "https://github.com/danswer-ai/danswer/blob/main/backend/alembic/README.md"} -->
+
+# Alembic DB Migrations
+These files are for creating/updating the tables in the Relational DB (Postgres).
+Danswer migrations use a generic single-database configuration with an async dbapi.
+
+## To generate new migrations: 
+run from danswer/backend:
+`alembic revision --autogenerate -m <DESCRIPTION_OF_MIGRATION>`
+
+More info can be found here: https://alembic.sqlalchemy.org/en/latest/autogenerate.html
+
+## Running migrations
+To run all un-applied migrations:
+`alembic upgrade head`
+
+To undo migrations:
+`alembic downgrade -X` 
+where X is the number of migrations you want to undo from the current state

backend/alembic/env.py

@@ -0,0 +1,90 @@
+import asyncio
+from logging.config import fileConfig
+
+from alembic import context
+from danswer.db.engine import build_connection_string
+from danswer.db.models import Base
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import create_async_engine
+from celery.backends.database.session import ResultModelBase  # type: ignore
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = [Base.metadata, ResultModelBase.metadata]
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = build_connection_string()
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,  # type: ignore
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+    context.configure(connection=connection, target_metadata=target_metadata)  # type: ignore
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+    """In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    connectable = create_async_engine(
+        build_connection_string(),
+        poolclass=pool.NullPool,
+    )
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+    await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode."""
+
+    asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

backend/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

backend/alembic/versions/0a2b51deb0b8_add_starter_prompts.py

@@ -0,0 +1,31 @@
+"""Add starter prompts
+
+Revision ID: 0a2b51deb0b8
+Revises: 5f4b8568a221
+Create Date: 2024-03-02 23:23:49.960309
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "0a2b51deb0b8"
+down_revision = "5f4b8568a221"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "persona",
+        sa.Column(
+            "starter_messages",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "starter_messages")

backend/alembic/versions/0a98909f2757_enable_encrypted_fields.py

@@ -0,0 +1,113 @@
+"""Enable Encrypted Fields
+
+Revision ID: 0a98909f2757
+Revises: 570282d33c49
+Create Date: 2024-05-05 19:30:34.317972
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import table
+from sqlalchemy.dialects import postgresql
+import json
+
+from danswer.utils.encryption import encrypt_string_to_bytes
+
+# revision identifiers, used by Alembic.
+revision = "0a98909f2757"
+down_revision = "570282d33c49"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    connection = op.get_bind()
+
+    op.alter_column("key_value_store", "value", nullable=True)
+    op.add_column(
+        "key_value_store",
+        sa.Column(
+            "encrypted_value",
+            sa.LargeBinary,
+            nullable=True,
+        ),
+    )
+
+    # Need a temporary column to translate the JSONB to binary
+    op.add_column("credential", sa.Column("temp_column", sa.LargeBinary()))
+
+    creds_table = table(
+        "credential",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "credential_json",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+        ),
+        sa.Column(
+            "temp_column",
+            sa.LargeBinary(),
+            nullable=False,
+        ),
+    )
+
+    results = connection.execute(sa.select(creds_table))
+
+    # This uses the MIT encrypt which does not actually encrypt the credentials
+    # In other words, this upgrade does not apply the encryption. Porting existing sensitive data
+    # and key rotation currently is not supported and will come out in the future
+    for row_id, creds, _ in results:
+        creds_binary = encrypt_string_to_bytes(json.dumps(creds))
+        connection.execute(
+            creds_table.update()
+            .where(creds_table.c.id == row_id)
+            .values(temp_column=creds_binary)
+        )
+
+    op.drop_column("credential", "credential_json")
+    op.alter_column("credential", "temp_column", new_column_name="credential_json")
+
+    op.add_column("llm_provider", sa.Column("temp_column", sa.LargeBinary()))
+
+    llm_table = table(
+        "llm_provider",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "api_key",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column(
+            "temp_column",
+            sa.LargeBinary(),
+            nullable=False,
+        ),
+    )
+    results = connection.execute(sa.select(llm_table))
+
+    for row_id, api_key, _ in results:
+        llm_key = encrypt_string_to_bytes(api_key)
+        connection.execute(
+            llm_table.update()
+            .where(llm_table.c.id == row_id)
+            .values(temp_column=llm_key)
+        )
+
+    op.drop_column("llm_provider", "api_key")
+    op.alter_column("llm_provider", "temp_column", new_column_name="api_key")
+
+
+def downgrade() -> None:
+    # Some information loss but this is ok. Should not allow decryption via downgrade.
+    op.drop_column("credential", "credential_json")
+    op.drop_column("llm_provider", "api_key")
+
+    op.add_column("llm_provider", sa.Column("api_key", sa.String()))
+    op.add_column(
+        "credential",
+        sa.Column("credential_json", postgresql.JSONB(astext_type=sa.Text())),
+    )
+
+    op.execute("DELETE FROM key_value_store WHERE value IS NULL")
+    op.alter_column("key_value_store", "value", nullable=False)
+    op.drop_column("key_value_store", "encrypted_value")

backend/alembic/versions/15326fcec57e_introduce_danswer_apis.py

@@ -0,0 +1,37 @@
+"""Introduce Danswer APIs
+
+Revision ID: 15326fcec57e
+Revises: 77d07dffae64
+Create Date: 2023-11-11 20:51:24.228999
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+from danswer.configs.constants import DocumentSource
+
+# revision identifiers, used by Alembic.
+revision = "15326fcec57e"
+down_revision = "77d07dffae64"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.alter_column("credential", "is_admin", new_column_name="admin_public")
+    op.add_column(
+        "document",
+        sa.Column("from_ingestion_api", sa.Boolean(), nullable=True),
+    )
+    op.alter_column(
+        "connector",
+        "source",
+        type_=sa.String(length=50),
+        existing_type=sa.Enum(DocumentSource, native_enum=False),
+        existing_nullable=False,
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("document", "from_ingestion_api")
+    op.alter_column("credential", "admin_public", new_column_name="is_admin")

backend/alembic/versions/173cae5bba26_port_config_store.py

@@ -0,0 +1,29 @@
+"""Port Config Store
+
+Revision ID: 173cae5bba26
+Revises: e50154680a5c
+Create Date: 2024-03-19 15:30:44.425436
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "173cae5bba26"
+down_revision = "e50154680a5c"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "key_value_store",
+        sa.Column("key", sa.String(), nullable=False),
+        sa.Column("value", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.PrimaryKeyConstraint("key"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("key_value_store")

backend/alembic/versions/2666d766cb9b_google_oauth2.py

@@ -0,0 +1,55 @@
+"""Google OAuth2
+
+Revision ID: 2666d766cb9b
+Revises: 6d387b3196c2
+Create Date: 2023-05-05 15:49:35.716016
+
+"""
+import fastapi_users_db_sqlalchemy
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "2666d766cb9b"
+down_revision = "6d387b3196c2"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "oauth_account",
+        sa.Column("id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=False,
+        ),
+        sa.Column("oauth_name", sa.String(length=100), nullable=False),
+        sa.Column("access_token", sa.String(length=1024), nullable=False),
+        sa.Column("expires_at", sa.Integer(), nullable=True),
+        sa.Column("refresh_token", sa.String(length=1024), nullable=True),
+        sa.Column("account_id", sa.String(length=320), nullable=False),
+        sa.Column("account_email", sa.String(length=320), nullable=False),
+        sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="cascade"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_oauth_account_account_id"),
+        "oauth_account",
+        ["account_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_oauth_account_oauth_name"),
+        "oauth_account",
+        ["oauth_name"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(op.f("ix_oauth_account_oauth_name"), table_name="oauth_account")
+    op.drop_index(op.f("ix_oauth_account_account_id"), table_name="oauth_account")
+    op.drop_table("oauth_account")

backend/alembic/versions/27c6ecc08586_permission_framework.py

@@ -0,0 +1,173 @@
+"""Permission Framework
+
+Revision ID: 27c6ecc08586
+Revises: 2666d766cb9b
+Create Date: 2023-05-24 18:45:17.244495
+
+"""
+import fastapi_users_db_sqlalchemy
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "27c6ecc08586"
+down_revision = "2666d766cb9b"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.execute("TRUNCATE TABLE index_attempt")
+    op.create_table(
+        "connector",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column(
+            "source",
+            sa.Enum(
+                "SLACK",
+                "WEB",
+                "GOOGLE_DRIVE",
+                "GITHUB",
+                "CONFLUENCE",
+                name="documentsource",
+                native_enum=False,
+            ),
+            nullable=False,
+        ),
+        sa.Column(
+            "input_type",
+            sa.Enum(
+                "LOAD_STATE",
+                "POLL",
+                "EVENT",
+                name="inputtype",
+                native_enum=False,
+            ),
+            nullable=True,
+        ),
+        sa.Column(
+            "connector_specific_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+        ),
+        sa.Column("refresh_freq", sa.Integer(), nullable=True),
+        sa.Column(
+            "time_created",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "time_updated",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column("disabled", sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "credential",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "credential_json",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+        ),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.Column("public_doc", sa.Boolean(), nullable=False),
+        sa.Column(
+            "time_created",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "time_updated",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "connector_credential_pair",
+        sa.Column("connector_id", sa.Integer(), nullable=False),
+        sa.Column("credential_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["connector_id"],
+            ["connector.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["credential_id"],
+            ["credential.id"],
+        ),
+        sa.PrimaryKeyConstraint("connector_id", "credential_id"),
+    )
+    op.add_column(
+        "index_attempt",
+        sa.Column("connector_id", sa.Integer(), nullable=True),
+    )
+    op.add_column(
+        "index_attempt",
+        sa.Column("credential_id", sa.Integer(), nullable=True),
+    )
+    op.create_foreign_key(
+        "fk_index_attempt_credential_id",
+        "index_attempt",
+        "credential",
+        ["credential_id"],
+        ["id"],
+    )
+    op.create_foreign_key(
+        "fk_index_attempt_connector_id",
+        "index_attempt",
+        "connector",
+        ["connector_id"],
+        ["id"],
+    )
+    op.drop_column("index_attempt", "connector_specific_config")
+    op.drop_column("index_attempt", "source")
+    op.drop_column("index_attempt", "input_type")
+
+
+def downgrade() -> None:
+    op.execute("TRUNCATE TABLE index_attempt")
+    op.add_column(
+        "index_attempt",
+        sa.Column("input_type", sa.VARCHAR(), autoincrement=False, nullable=False),
+    )
+    op.add_column(
+        "index_attempt",
+        sa.Column("source", sa.VARCHAR(), autoincrement=False, nullable=False),
+    )
+    op.add_column(
+        "index_attempt",
+        sa.Column(
+            "connector_specific_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            autoincrement=False,
+            nullable=False,
+        ),
+    )
+    op.drop_constraint(
+        "fk_index_attempt_credential_id", "index_attempt", type_="foreignkey"
+    )
+    op.drop_constraint(
+        "fk_index_attempt_connector_id", "index_attempt", type_="foreignkey"
+    )
+    op.drop_column("index_attempt", "credential_id")
+    op.drop_column("index_attempt", "connector_id")
+    op.drop_table("connector_credential_pair")
+    op.drop_table("credential")
+    op.drop_table("connector")

backend/alembic/versions/30c1d5744104_persona_datetime_aware.py

@@ -0,0 +1,37 @@
+"""Persona Datetime Aware
+
+Revision ID: 30c1d5744104
+Revises: 7f99be1cb9f5
+Create Date: 2023-10-16 23:21:01.283424
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "30c1d5744104"
+down_revision = "7f99be1cb9f5"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column("persona", sa.Column("datetime_aware", sa.Boolean(), nullable=True))
+    op.execute("UPDATE persona SET datetime_aware = TRUE")
+    op.alter_column("persona", "datetime_aware", nullable=False)
+    op.create_index(
+        "_default_persona_name_idx",
+        "persona",
+        ["name"],
+        unique=True,
+        postgresql_where=sa.text("default_persona = true"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "_default_persona_name_idx",
+        table_name="persona",
+        postgresql_where=sa.text("default_persona = true"),
+    )
+    op.drop_column("persona", "datetime_aware")

backend/alembic/versions/3879338f8ba1_add_tool_table.py

@@ -0,0 +1,45 @@
+"""Add tool table
+
+Revision ID: 3879338f8ba1
+Revises: f1c6478c3fd8
+Create Date: 2024-05-11 16:11:23.718084
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "3879338f8ba1"
+down_revision = "f1c6478c3fd8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "tool",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("in_code_tool_id", sa.String(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "persona__tool",
+        sa.Column("persona_id", sa.Integer(), nullable=False),
+        sa.Column("tool_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["persona_id"],
+            ["persona.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["tool_id"],
+            ["tool.id"],
+        ),
+        sa.PrimaryKeyConstraint("persona_id", "tool_id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("persona__tool")
+    op.drop_table("tool")

backend/alembic/versions/38eda64af7fe_add_chat_session_sharing.py

@@ -0,0 +1,41 @@
+"""Add chat session sharing
+
+Revision ID: 38eda64af7fe
+Revises: 776b3bbe9092
+Create Date: 2024-03-27 19:41:29.073594
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "38eda64af7fe"
+down_revision = "776b3bbe9092"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_session",
+        sa.Column(
+            "shared_status",
+            sa.Enum(
+                "PUBLIC",
+                "PRIVATE",
+                name="chatsessionsharedstatus",
+                native_enum=False,
+            ),
+            nullable=True,
+        ),
+    )
+    op.execute("UPDATE chat_session SET shared_status='PRIVATE'")
+    op.alter_column(
+        "chat_session",
+        "shared_status",
+        nullable=False,
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("chat_session", "shared_status")

backend/alembic/versions/3b25685ff73c_move_is_public_to_cc_pair.py

@@ -0,0 +1,49 @@
+"""Move is_public to cc_pair
+
+Revision ID: 3b25685ff73c
+Revises: e0a68a81d434
+Create Date: 2023-10-05 18:47:09.582849
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "3b25685ff73c"
+down_revision = "e0a68a81d434"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "connector_credential_pair",
+        sa.Column("is_public", sa.Boolean(), nullable=True),
+    )
+    # fill in is_public for existing rows
+    op.execute(
+        "UPDATE connector_credential_pair SET is_public = true WHERE is_public IS NULL"
+    )
+    op.alter_column("connector_credential_pair", "is_public", nullable=False)
+
+    op.add_column(
+        "credential",
+        sa.Column("is_admin", sa.Boolean(), nullable=True),
+    )
+    op.execute("UPDATE credential SET is_admin = true WHERE is_admin IS NULL")
+    op.alter_column("credential", "is_admin", nullable=False)
+
+    op.drop_column("credential", "public_doc")
+
+
+def downgrade() -> None:
+    op.add_column(
+        "credential",
+        sa.Column("public_doc", sa.Boolean(), nullable=True),
+    )
+    # setting public_doc to false for all existing rows to be safe
+    # NOTE: this is likely not the correct state of the world but it's the best we can do
+    op.execute("UPDATE credential SET public_doc = false WHERE public_doc IS NULL")
+    op.alter_column("credential", "public_doc", nullable=False)
+    op.drop_column("connector_credential_pair", "is_public")
+    op.drop_column("credential", "is_admin")

backend/alembic/versions/3c5e35aa9af0_polling_document_count.py

@@ -0,0 +1,52 @@
+"""Polling Document Count
+
+Revision ID: 3c5e35aa9af0
+Revises: 27c6ecc08586
+Create Date: 2023-06-14 23:45:51.760440
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "3c5e35aa9af0"
+down_revision = "27c6ecc08586"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "connector_credential_pair",
+        sa.Column(
+            "last_successful_index_time",
+            sa.DateTime(timezone=True),
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "connector_credential_pair",
+        sa.Column(
+            "last_attempt_status",
+            sa.Enum(
+                "NOT_STARTED",
+                "IN_PROGRESS",
+                "SUCCESS",
+                "FAILED",
+                name="indexingstatus",
+                native_enum=False,
+            ),
+            nullable=False,
+        ),
+    )
+    op.add_column(
+        "connector_credential_pair",
+        sa.Column("total_docs_indexed", sa.Integer(), nullable=False),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("connector_credential_pair", "total_docs_indexed")
+    op.drop_column("connector_credential_pair", "last_attempt_status")
+    op.drop_column("connector_credential_pair", "last_successful_index_time")

backend/alembic/versions/401c1ac29467_add_tables_for_ui_based_llm_.py

@@ -0,0 +1,49 @@
+"""Add tables for UI-based LLM configuration
+
+Revision ID: 401c1ac29467
+Revises: 703313b75876
+Create Date: 2024-04-13 18:07:29.153817
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "401c1ac29467"
+down_revision = "703313b75876"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "llm_provider",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("api_key", sa.String(), nullable=True),
+        sa.Column("api_base", sa.String(), nullable=True),
+        sa.Column("api_version", sa.String(), nullable=True),
+        sa.Column(
+            "custom_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+        sa.Column("default_model_name", sa.String(), nullable=False),
+        sa.Column("fast_default_model_name", sa.String(), nullable=True),
+        sa.Column("is_default_provider", sa.Boolean(), unique=True, nullable=True),
+        sa.Column("model_names", postgresql.ARRAY(sa.String()), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
+    )
+
+    op.add_column(
+        "persona",
+        sa.Column("llm_model_provider_override", sa.String(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "llm_model_provider_override")
+
+    op.drop_table("llm_provider")

backend/alembic/versions/465f78d9b7f9_larger_access_tokens_for_oauth.py

@@ -0,0 +1,24 @@
+"""Larger Access Tokens for OAUTH
+
+Revision ID: 465f78d9b7f9
+Revises: 3c5e35aa9af0
+Create Date: 2023-07-18 17:33:40.365034
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "465f78d9b7f9"
+down_revision = "3c5e35aa9af0"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.alter_column("oauth_account", "access_token", type_=sa.Text())
+
+
+def downgrade() -> None:
+    op.alter_column("oauth_account", "access_token", type_=sa.String(length=1024))

backend/alembic/versions/46625e4745d4_remove_native_enum.py

@@ -0,0 +1,31 @@
+"""Remove Native Enum
+
+Revision ID: 46625e4745d4
+Revises: 9d97fecfab7f
+Create Date: 2023-10-27 11:38:33.803145
+
+"""
+from alembic import op
+from sqlalchemy import String
+
+# revision identifiers, used by Alembic.
+revision = "46625e4745d4"
+down_revision = "9d97fecfab7f"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    # At this point, we directly changed some previous migrations,
+    # https://github.com/danswer-ai/danswer/pull/637
+    # Due to using Postgres native Enums, it caused some complications for first time users.
+    # To remove those complications, all Enums are only handled application side moving forward.
+    # This migration exists to ensure that existing users don't run into upgrade issues.
+    op.alter_column("index_attempt", "status", type_=String)
+    op.alter_column("connector_credential_pair", "last_attempt_status", type_=String)
+    op.execute("DROP TYPE IF EXISTS indexingstatus")
+
+
+def downgrade() -> None:
+    # We don't want Native Enums, do nothing
+    pass

backend/alembic/versions/4738e4b3bae1_pg_file_store.py

@@ -0,0 +1,28 @@
+"""PG File Store
+
+Revision ID: 4738e4b3bae1
+Revises: e91df4e935ef
+Create Date: 2024-03-20 18:53:32.461518
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "4738e4b3bae1"
+down_revision = "e91df4e935ef"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "file_store",
+        sa.Column("file_name", sa.String(), nullable=False),
+        sa.Column("lobj_oid", sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint("file_name"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("file_store")

backend/alembic/versions/47433d30de82_create_indexattempt_table.py

@@ -0,0 +1,73 @@
+"""Create IndexAttempt table
+
+Revision ID: 47433d30de82
+Revises:
+Create Date: 2023-05-04 00:55:32.971991
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "47433d30de82"
+down_revision: None = None
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "index_attempt",
+        sa.Column("id", sa.Integer(), nullable=False),
+        # String type since python enum will change often
+        sa.Column(
+            "source",
+            sa.String(),
+            nullable=False,
+        ),
+        # String type to easily accomodate new ways of pulling
+        # in documents
+        sa.Column(
+            "input_type",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column(
+            "connector_specific_config",
+            postgresql.JSONB(),
+            nullable=False,
+        ),
+        sa.Column(
+            "time_created",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "time_updated",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            server_onupdate=sa.text("now()"),  # type: ignore
+            nullable=True,
+        ),
+        sa.Column(
+            "status",
+            sa.Enum(
+                "NOT_STARTED",
+                "IN_PROGRESS",
+                "SUCCESS",
+                "FAILED",
+                name="indexingstatus",
+                native_enum=False,
+            ),
+            nullable=False,
+        ),
+        sa.Column("document_ids", postgresql.ARRAY(sa.String()), nullable=True),
+        sa.Column("error_msg", sa.String(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("index_attempt")

backend/alembic/versions/475fcefe8826_add_name_to_api_key.py

@@ -0,0 +1,23 @@
+"""Add name to api_key
+
+Revision ID: 475fcefe8826
+Revises: ecab2b3f1a3b
+Create Date: 2024-04-11 11:05:18.414438
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "475fcefe8826"
+down_revision = "ecab2b3f1a3b"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column("api_key", sa.Column("name", sa.String(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("api_key", "name")

backend/alembic/versions/50b683a8295c_add_additional_retrieval_controls_to_.py

@@ -0,0 +1,28 @@
+"""Add additional retrieval controls to Persona
+
+Revision ID: 50b683a8295c
+Revises: 7da0ae5ad583
+Create Date: 2023-11-27 17:23:29.668422
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "50b683a8295c"
+down_revision = "7da0ae5ad583"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column("persona", sa.Column("num_chunks", sa.Integer(), nullable=True))
+    op.add_column(
+        "persona",
+        sa.Column("apply_llm_relevance_filter", sa.Boolean(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "apply_llm_relevance_filter")
+    op.drop_column("persona", "num_chunks")

backend/alembic/versions/570282d33c49_track_danswerbot_explicitly.py

@@ -0,0 +1,27 @@
+"""Track Danswerbot Explicitly
+
+Revision ID: 570282d33c49
+Revises: 7547d982db8f
+Create Date: 2024-05-04 17:49:28.568109
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "570282d33c49"
+down_revision = "7547d982db8f"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_session", sa.Column("danswerbot_flow", sa.Boolean(), nullable=True)
+    )
+    op.execute("UPDATE chat_session SET danswerbot_flow = one_shot")
+    op.alter_column("chat_session", "danswerbot_flow", nullable=False)
+
+
+def downgrade() -> None:
+    op.drop_column("chat_session", "danswerbot_flow")

backend/alembic/versions/57b53544726e_add_document_set_tables.py

@@ -0,0 +1,59 @@
+"""Add document set tables
+
+Revision ID: 57b53544726e
+Revises: 800f48024ae9
+Create Date: 2023-09-20 16:59:39.097177
+
+"""
+from alembic import op
+import fastapi_users_db_sqlalchemy
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "57b53544726e"
+down_revision = "800f48024ae9"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "document_set",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("description", sa.String(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.Column("is_up_to_date", sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
+    )
+    op.create_table(
+        "document_set__connector_credential_pair",
+        sa.Column("document_set_id", sa.Integer(), nullable=False),
+        sa.Column("connector_credential_pair_id", sa.Integer(), nullable=False),
+        sa.Column("is_current", sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["connector_credential_pair_id"],
+            ["connector_credential_pair.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["document_set_id"],
+            ["document_set.id"],
+        ),
+        sa.PrimaryKeyConstraint(
+            "document_set_id", "connector_credential_pair_id", "is_current"
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("document_set__connector_credential_pair")
+    op.drop_table("document_set")

backend/alembic/versions/5809c0787398_add_chat_sessions.py

@@ -0,0 +1,85 @@
+"""Add Chat Sessions
+
+Revision ID: 5809c0787398
+Revises: d929f0c1c6af
+Create Date: 2023-09-04 15:29:44.002164
+
+"""
+import fastapi_users_db_sqlalchemy
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "5809c0787398"
+down_revision = "d929f0c1c6af"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "chat_session",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.Column("description", sa.Text(), nullable=False),
+        sa.Column("deleted", sa.Boolean(), nullable=False),
+        sa.Column(
+            "time_updated",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "time_created",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "chat_message",
+        sa.Column("chat_session_id", sa.Integer(), nullable=False),
+        sa.Column("message_number", sa.Integer(), nullable=False),
+        sa.Column("edit_number", sa.Integer(), nullable=False),
+        sa.Column("parent_edit_number", sa.Integer(), nullable=True),
+        sa.Column("latest", sa.Boolean(), nullable=False),
+        sa.Column("message", sa.Text(), nullable=False),
+        sa.Column(
+            "message_type",
+            sa.Enum(
+                "SYSTEM",
+                "USER",
+                "ASSISTANT",
+                "DANSWER",
+                name="messagetype",
+                native_enum=False,
+            ),
+            nullable=False,
+        ),
+        sa.Column(
+            "time_sent",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["chat_session_id"],
+            ["chat_session.id"],
+        ),
+        sa.PrimaryKeyConstraint("chat_session_id", "message_number", "edit_number"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("chat_message")
+    op.drop_table("chat_session")

backend/alembic/versions/5e84129c8be3_add_docs_indexed_column_to_index_.py

@@ -0,0 +1,36 @@
+"""Add docs_indexed_column + time_started to index_attempt table
+
+Revision ID: 5e84129c8be3
+Revises: e6a4bbc13fe4
+Create Date: 2023-08-10 21:43:09.069523
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "5e84129c8be3"
+down_revision = "e6a4bbc13fe4"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "index_attempt",
+        sa.Column("num_docs_indexed", sa.Integer()),
+    )
+    op.add_column(
+        "index_attempt",
+        sa.Column(
+            "time_started",
+            sa.DateTime(timezone=True),
+            nullable=True,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("index_attempt", "time_started")
+    op.drop_column("index_attempt", "num_docs_indexed")

backend/alembic/versions/5f4b8568a221_add_removed_documents_to_index_attempt.py

@@ -0,0 +1,27 @@
+"""add removed documents to index_attempt
+
+Revision ID: 5f4b8568a221
+Revises: dbaa756c2ccf
+Create Date: 2024-02-16 15:02:03.319907
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "5f4b8568a221"
+down_revision = "8987770549c0"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "index_attempt",
+        sa.Column("docs_removed_from_index", sa.Integer()),
+    )
+    op.execute("UPDATE index_attempt SET docs_removed_from_index = 0")
+
+
+def downgrade() -> None:
+    op.drop_column("index_attempt", "docs_removed_from_index")

backend/alembic/versions/643a84a42a33_add_user_configured_names_to_llmprovider.py

@@ -0,0 +1,45 @@
+"""Add user-configured names to LLMProvider
+
+Revision ID: 643a84a42a33
+Revises: 0a98909f2757
+Create Date: 2024-05-07 14:54:55.493100
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "643a84a42a33"
+down_revision = "0a98909f2757"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column("llm_provider", sa.Column("provider", sa.String(), nullable=True))
+    # move "name" -> "provider" to match the new schema
+    op.execute("UPDATE llm_provider SET provider = name")
+    # pretty up display name
+    op.execute("UPDATE llm_provider SET name = 'OpenAI' WHERE name = 'openai'")
+    op.execute("UPDATE llm_provider SET name = 'Anthropic' WHERE name = 'anthropic'")
+    op.execute("UPDATE llm_provider SET name = 'Azure OpenAI' WHERE name = 'azure'")
+    op.execute("UPDATE llm_provider SET name = 'AWS Bedrock' WHERE name = 'bedrock'")
+
+    # update personas to use the new provider names
+    op.execute(
+        "UPDATE persona SET llm_model_provider_override = 'OpenAI' WHERE llm_model_provider_override = 'openai'"
+    )
+    op.execute(
+        "UPDATE persona SET llm_model_provider_override = 'Anthropic' WHERE llm_model_provider_override = 'anthropic'"
+    )
+    op.execute(
+        "UPDATE persona SET llm_model_provider_override = 'Azure OpenAI' WHERE llm_model_provider_override = 'azure'"
+    )
+    op.execute(
+        "UPDATE persona SET llm_model_provider_override = 'AWS Bedrock' WHERE llm_model_provider_override = 'bedrock'"
+    )
+
+
+def downgrade() -> None:
+    op.execute("UPDATE llm_provider SET name = provider")
+    op.drop_column("llm_provider", "provider")

backend/alembic/versions/6d387b3196c2_basic_auth.py

@@ -0,0 +1,92 @@
+"""Basic Auth
+
+Revision ID: 6d387b3196c2
+Revises: 47433d30de82
+Create Date: 2023-05-05 14:40:10.242502
+
+"""
+import fastapi_users_db_sqlalchemy
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "6d387b3196c2"
+down_revision = "47433d30de82"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "user",
+        sa.Column("id", fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False),
+        sa.Column("email", sa.String(length=320), nullable=False),
+        sa.Column("hashed_password", sa.String(length=1024), nullable=False),
+        sa.Column("is_active", sa.Boolean(), nullable=False),
+        sa.Column("is_superuser", sa.Boolean(), nullable=False),
+        sa.Column("is_verified", sa.Boolean(), nullable=False),
+        sa.Column(
+            "role",
+            sa.Enum("BASIC", "ADMIN", name="userrole", native_enum=False),
+            default="BASIC",
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_user_email"), "user", ["email"], unique=True)
+    op.create_table(
+        "accesstoken",
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=False,
+        ),
+        sa.Column("token", sa.String(length=43), nullable=False),
+        sa.Column(
+            "created_at",
+            fastapi_users_db_sqlalchemy.generics.TIMESTAMPAware(timezone=True),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="cascade"),
+        sa.PrimaryKeyConstraint("token"),
+    )
+    op.create_index(
+        op.f("ix_accesstoken_created_at"),
+        "accesstoken",
+        ["created_at"],
+        unique=False,
+    )
+    op.alter_column(
+        "index_attempt",
+        "time_created",
+        existing_type=postgresql.TIMESTAMP(timezone=True),
+        nullable=False,
+        existing_server_default=sa.text("now()"),  # type: ignore
+    )
+    op.alter_column(
+        "index_attempt",
+        "time_updated",
+        existing_type=postgresql.TIMESTAMP(timezone=True),
+        nullable=False,
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "index_attempt",
+        "time_updated",
+        existing_type=postgresql.TIMESTAMP(timezone=True),
+        nullable=True,
+    )
+    op.alter_column(
+        "index_attempt",
+        "time_created",
+        existing_type=postgresql.TIMESTAMP(timezone=True),
+        nullable=True,
+        existing_server_default=sa.text("now()"),  # type: ignore
+    )
+    op.drop_index(op.f("ix_accesstoken_created_at"), table_name="accesstoken")
+    op.drop_table("accesstoken")
+    op.drop_index(op.f("ix_user_email"), table_name="user")
+    op.drop_table("user")

backend/alembic/versions/703313b75876_add_tokenratelimit_tables.py

@@ -0,0 +1,83 @@
+"""Add TokenRateLimit Tables
+
+Revision ID: 703313b75876
+Revises: fad14119fb92
+Create Date: 2024-04-15 01:36:02.952809
+
+"""
+import json
+from typing import cast
+from alembic import op
+import sqlalchemy as sa
+from danswer.dynamic_configs.factory import get_dynamic_config_store
+
+# revision identifiers, used by Alembic.
+revision = "703313b75876"
+down_revision = "fad14119fb92"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "token_rate_limit",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("enabled", sa.Boolean(), nullable=False),
+        sa.Column("token_budget", sa.Integer(), nullable=False),
+        sa.Column("period_hours", sa.Integer(), nullable=False),
+        sa.Column(
+            "scope",
+            sa.String(length=10),
+            nullable=False,
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "token_rate_limit__user_group",
+        sa.Column("rate_limit_id", sa.Integer(), nullable=False),
+        sa.Column("user_group_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["rate_limit_id"],
+            ["token_rate_limit.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_group_id"],
+            ["user_group.id"],
+        ),
+        sa.PrimaryKeyConstraint("rate_limit_id", "user_group_id"),
+    )
+
+    try:
+        settings_json = cast(
+            str, get_dynamic_config_store().load("token_budget_settings")
+        )
+        settings = json.loads(settings_json)
+
+        is_enabled = settings.get("enable_token_budget", False)
+        token_budget = settings.get("token_budget", -1)
+        period_hours = settings.get("period_hours", -1)
+
+        if is_enabled and token_budget > 0 and period_hours > 0:
+            op.execute(
+                f"INSERT INTO token_rate_limit \
+                    (enabled, token_budget, period_hours, scope) VALUES \
+                        ({is_enabled}, {token_budget}, {period_hours}, 'GLOBAL')"
+            )
+
+        # Delete the dynamic config
+        get_dynamic_config_store().delete("token_budget_settings")
+
+    except Exception:
+        # Ignore if the dynamic config is not found
+        pass
+
+
+def downgrade() -> None:
+    op.drop_table("token_rate_limit__user_group")
+    op.drop_table("token_rate_limit")

backend/alembic/versions/72bdc9929a46_permission_auto_sync_framework.py

@@ -0,0 +1,81 @@
+"""Permission Auto Sync Framework
+
+Revision ID: 72bdc9929a46
+Revises: 475fcefe8826
+Create Date: 2024-04-14 21:15:28.659634
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "72bdc9929a46"
+down_revision = "475fcefe8826"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "email_to_external_user_cache",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("external_user_id", sa.String(), nullable=False),
+        sa.Column("user_id", sa.UUID(), nullable=True),
+        sa.Column("user_email", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "external_permission",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.UUID(), nullable=True),
+        sa.Column("user_email", sa.String(), nullable=False),
+        sa.Column(
+            "source_type",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column("external_permission_group", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "permission_sync_run",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "source_type",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column("update_type", sa.String(), nullable=False),
+        sa.Column("cc_pair_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "status",
+            sa.String(),
+            nullable=False,
+        ),
+        sa.Column("error_msg", sa.Text(), nullable=True),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["cc_pair_id"],
+            ["connector_credential_pair.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("permission_sync_run")
+    op.drop_table("external_permission")
+    op.drop_table("email_to_external_user_cache")

backend/alembic/versions/7547d982db8f_chat_folders.py

@@ -0,0 +1,51 @@
+"""Chat Folders
+
+Revision ID: 7547d982db8f
+Revises: ef7da92f7213
+Create Date: 2024-05-02 15:18:56.573347
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import fastapi_users_db_sqlalchemy
+
+# revision identifiers, used by Alembic.
+revision = "7547d982db8f"
+down_revision = "ef7da92f7213"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "chat_folder",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column("display_priority", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.add_column("chat_session", sa.Column("folder_id", sa.Integer(), nullable=True))
+    op.create_foreign_key(
+        "chat_session_chat_folder_fk",
+        "chat_session",
+        "chat_folder",
+        ["folder_id"],
+        ["id"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "chat_session_chat_folder_fk", "chat_session", type_="foreignkey"
+    )
+    op.drop_column("chat_session", "folder_id")
+    op.drop_table("chat_folder")

backend/alembic/versions/767f1c2a00eb_count_chat_tokens.py

@@ -0,0 +1,26 @@
+"""Count Chat Tokens
+
+Revision ID: 767f1c2a00eb
+Revises: dba7f71618f5
+Create Date: 2023-09-21 10:03:21.509899
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "767f1c2a00eb"
+down_revision = "dba7f71618f5"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_message", sa.Column("token_count", sa.Integer(), nullable=False)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("chat_message", "token_count")

backend/alembic/versions/76b60d407dfb_cc_pair_name_not_unique.py

@@ -0,0 +1,32 @@
+"""CC-Pair Name not Unique
+
+Revision ID: 76b60d407dfb
+Revises: b156fa702355
+Create Date: 2023-12-22 21:42:10.018804
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "76b60d407dfb"
+down_revision = "b156fa702355"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.execute("DELETE FROM connector_credential_pair WHERE name IS NULL")
+    op.drop_constraint(
+        "connector_credential_pair__name__key",
+        "connector_credential_pair",
+        type_="unique",
+    )
+    op.alter_column(
+        "connector_credential_pair", "name", existing_type=sa.String(), nullable=False
+    )
+
+
+def downgrade() -> None:
+    # This wasn't really required by the code either, no good reason to make it unique again
+    pass

backend/alembic/versions/776b3bbe9092_remove_remaining_enums.py

@@ -0,0 +1,71 @@
+"""Remove Remaining Enums
+
+Revision ID: 776b3bbe9092
+Revises: 4738e4b3bae1
+Create Date: 2024-03-22 21:34:27.629444
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+from danswer.db.models import IndexModelStatus
+from danswer.search.enums import RecencyBiasSetting
+from danswer.search.models import SearchType
+
+# revision identifiers, used by Alembic.
+revision = "776b3bbe9092"
+down_revision = "4738e4b3bae1"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "persona",
+        "search_type",
+        type_=sa.String,
+        existing_type=sa.Enum(SearchType, native_enum=False),
+        existing_nullable=False,
+    )
+    op.alter_column(
+        "persona",
+        "recency_bias",
+        type_=sa.String,
+        existing_type=sa.Enum(RecencyBiasSetting, native_enum=False),
+        existing_nullable=False,
+    )
+
+    # Because the indexmodelstatus enum does not have a mapping to a string type
+    # we need this workaround instead of directly changing the type
+    op.add_column("embedding_model", sa.Column("temp_status", sa.String))
+    op.execute("UPDATE embedding_model SET temp_status = status::text")
+    op.drop_column("embedding_model", "status")
+    op.alter_column("embedding_model", "temp_status", new_column_name="status")
+
+    op.execute("DROP TYPE IF EXISTS searchtype")
+    op.execute("DROP TYPE IF EXISTS recencybiassetting")
+    op.execute("DROP TYPE IF EXISTS indexmodelstatus")
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "persona",
+        "search_type",
+        type_=sa.Enum(SearchType, native_enum=False),
+        existing_type=sa.String(length=50),
+        existing_nullable=False,
+    )
+    op.alter_column(
+        "persona",
+        "recency_bias",
+        type_=sa.Enum(RecencyBiasSetting, native_enum=False),
+        existing_type=sa.String(length=50),
+        existing_nullable=False,
+    )
+    op.alter_column(
+        "embedding_model",
+        "status",
+        type_=sa.Enum(IndexModelStatus, native_enum=False),
+        existing_type=sa.String(length=50),
+        existing_nullable=False,
+    )

backend/alembic/versions/77d07dffae64_forcibly_remove_more_enum_types_from_.py

@@ -0,0 +1,35 @@
+"""forcibly remove more enum types from postgres
+
+Revision ID: 77d07dffae64
+Revises: d61e513bef0a
+Create Date: 2023-11-01 12:33:01.999617
+
+"""
+from alembic import op
+from sqlalchemy import String
+
+
+# revision identifiers, used by Alembic.
+revision = "77d07dffae64"
+down_revision = "d61e513bef0a"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    # In a PR:
+    # https://github.com/danswer-ai/danswer/pull/397/files#diff-f05fb341f6373790b91852579631b64ca7645797a190837156a282b67e5b19c2
+    # we directly changed some previous migrations. This caused some users to have native enums
+    # while others wouldn't. This has caused some issues when adding new fields to these enums.
+    # This migration manually changes the enum types to ensure that nobody uses native enums.
+    op.alter_column("query_event", "selected_search_flow", type_=String)
+    op.alter_column("query_event", "feedback", type_=String)
+    op.alter_column("document_retrieval_feedback", "feedback", type_=String)
+    op.execute("DROP TYPE IF EXISTS searchtype")
+    op.execute("DROP TYPE IF EXISTS qafeedbacktype")
+    op.execute("DROP TYPE IF EXISTS searchfeedbacktype")
+
+
+def downgrade() -> None:
+    # We don't want Native Enums, do nothing
+    pass

backend/alembic/versions/78dbe7e38469_task_tracking.py

@@ -0,0 +1,48 @@
+"""Task Tracking
+
+Revision ID: 78dbe7e38469
+Revises: 7ccea01261f6
+Create Date: 2023-10-15 23:40:50.593262
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "78dbe7e38469"
+down_revision = "7ccea01261f6"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "task_queue_jobs",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("task_id", sa.String(), nullable=False),
+        sa.Column("task_name", sa.String(), nullable=False),
+        sa.Column(
+            "status",
+            sa.Enum(
+                "PENDING",
+                "STARTED",
+                "SUCCESS",
+                "FAILURE",
+                name="taskstatus",
+                native_enum=False,
+            ),
+            nullable=False,
+        ),
+        sa.Column("start_time", sa.DateTime(timezone=True), nullable=True),
+        sa.Column(
+            "register_time",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("task_queue_jobs")

backend/alembic/versions/79acd316403a_add_api_key_table.py

@@ -0,0 +1,48 @@
+"""Add api_key table
+
+Revision ID: 79acd316403a
+Revises: 904e5138fffb
+Create Date: 2024-01-11 17:56:37.934381
+
+"""
+from alembic import op
+import fastapi_users_db_sqlalchemy
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "79acd316403a"
+down_revision = "904e5138fffb"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "api_key",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("hashed_api_key", sa.String(), nullable=False),
+        sa.Column("api_key_display", sa.String(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=False,
+        ),
+        sa.Column(
+            "owner_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("api_key_display"),
+        sa.UniqueConstraint("hashed_api_key"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("api_key")

backend/alembic/versions/7ccea01261f6_store_chat_retrieval_docs.py

@@ -0,0 +1,31 @@
+"""Store Chat Retrieval Docs
+
+Revision ID: 7ccea01261f6
+Revises: a570b80a5f20
+Create Date: 2023-10-15 10:39:23.317453
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "7ccea01261f6"
+down_revision = "a570b80a5f20"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_message",
+        sa.Column(
+            "reference_docs",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("chat_message", "reference_docs")

backend/alembic/versions/7da0ae5ad583_add_description_to_persona.py

@@ -0,0 +1,23 @@
+"""Add description to persona
+
+Revision ID: 7da0ae5ad583
+Revises: e86866a9c78a
+Create Date: 2023-11-27 00:16:19.959414
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "7da0ae5ad583"
+down_revision = "e86866a9c78a"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column("persona", sa.Column("description", sa.String(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "description")

backend/alembic/versions/7da543f5672f_add_slackbotconfig_table.py

@@ -0,0 +1,38 @@
+"""Add SlackBotConfig table
+
+Revision ID: 7da543f5672f
+Revises: febe9eaa0644
+Create Date: 2023-09-24 16:34:17.526128
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "7da543f5672f"
+down_revision = "febe9eaa0644"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "slack_bot_config",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("persona_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "channel_config",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["persona_id"],
+            ["persona.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("slack_bot_config")

backend/alembic/versions/7f726bad5367_slack_followup.py

@@ -0,0 +1,26 @@
+"""Slack Followup
+
+Revision ID: 7f726bad5367
+Revises: 79acd316403a
+Create Date: 2024-01-15 00:19:55.991224
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "7f726bad5367"
+down_revision = "79acd316403a"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_feedback",
+        sa.Column("required_followup", sa.Boolean(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("chat_feedback", "required_followup")

backend/alembic/versions/7f99be1cb9f5_add_index_for_getting_documents_just_by_.py

@@ -0,0 +1,35 @@
+"""Add index for getting documents just by connector id / credential id
+
+Revision ID: 7f99be1cb9f5
+Revises: 78dbe7e38469
+Create Date: 2023-10-15 22:48:15.487762
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "7f99be1cb9f5"
+down_revision = "78dbe7e38469"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_index(
+        op.f(
+            "ix_document_by_connector_credential_pair_pkey__connector_id__credential_id"
+        ),
+        "document_by_connector_credential_pair",
+        ["connector_id", "credential_id"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        op.f(
+            "ix_document_by_connector_credential_pair_pkey__connector_id__credential_id"
+        ),
+        table_name="document_by_connector_credential_pair",
+    )

backend/alembic/versions/800f48024ae9_add_id_to_connectorcredentialpair.py

@@ -0,0 +1,60 @@
+"""Add ID to ConnectorCredentialPair
+
+Revision ID: 800f48024ae9
+Revises: 767f1c2a00eb
+Create Date: 2023-09-19 16:13:42.299715
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.schema import Sequence, CreateSequence
+
+# revision identifiers, used by Alembic.
+revision = "800f48024ae9"
+down_revision = "767f1c2a00eb"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    sequence = Sequence("connector_credential_pair_id_seq")
+    op.execute(CreateSequence(sequence))  # type: ignore
+    op.add_column(
+        "connector_credential_pair",
+        sa.Column(
+            "id", sa.Integer(), nullable=True, server_default=sequence.next_value()
+        ),
+    )
+    op.add_column(
+        "connector_credential_pair",
+        sa.Column("name", sa.String(), nullable=True),
+    )
+
+    # fill in IDs for existing rows
+    op.execute(
+        "UPDATE connector_credential_pair SET id = nextval('connector_credential_pair_id_seq') WHERE id IS NULL"
+    )
+    op.alter_column("connector_credential_pair", "id", nullable=False)
+
+    op.create_unique_constraint(
+        "connector_credential_pair__name__key", "connector_credential_pair", ["name"]
+    )
+    op.create_unique_constraint(
+        "connector_credential_pair__id__key", "connector_credential_pair", ["id"]
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "connector_credential_pair__name__key",
+        "connector_credential_pair",
+        type_="unique",
+    )
+    op.drop_constraint(
+        "connector_credential_pair__id__key",
+        "connector_credential_pair",
+        type_="unique",
+    )
+    op.drop_column("connector_credential_pair", "name")
+    op.drop_column("connector_credential_pair", "id")
+    op.execute("DROP SEQUENCE connector_credential_pair_id_seq")

backend/alembic/versions/80696cf850ae_add_chat_session_to_query_event.py

@@ -0,0 +1,36 @@
+"""Add chat session to query_event
+
+Revision ID: 80696cf850ae
+Revises: 15326fcec57e
+Create Date: 2023-11-26 02:38:35.008070
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "80696cf850ae"
+down_revision = "15326fcec57e"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "query_event",
+        sa.Column("chat_session_id", sa.Integer(), nullable=True),
+    )
+    op.create_foreign_key(
+        "fk_query_event_chat_session_id",
+        "query_event",
+        "chat_session",
+        ["chat_session_id"],
+        ["id"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "fk_query_event_chat_session_id", "query_event", type_="foreignkey"
+    )
+    op.drop_column("query_event", "chat_session_id")

backend/alembic/versions/891cd83c87a8_add_is_visible_to_persona.py

@@ -0,0 +1,34 @@
+"""Add is_visible to Persona
+
+Revision ID: 891cd83c87a8
+Revises: 76b60d407dfb
+Create Date: 2023-12-21 11:55:54.132279
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "891cd83c87a8"
+down_revision = "76b60d407dfb"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "persona",
+        sa.Column("is_visible", sa.Boolean(), nullable=True),
+    )
+    op.execute("UPDATE persona SET is_visible = true")
+    op.alter_column("persona", "is_visible", nullable=False)
+
+    op.add_column(
+        "persona",
+        sa.Column("display_priority", sa.Integer(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "is_visible")
+    op.drop_column("persona", "display_priority")

backend/alembic/versions/8987770549c0_add_full_exception_stack_trace.py

@@ -0,0 +1,25 @@
+"""Add full exception stack trace
+
+Revision ID: 8987770549c0
+Revises: ec3ec2eabf7b
+Create Date: 2024-02-10 19:31:28.339135
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "8987770549c0"
+down_revision = "ec3ec2eabf7b"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "index_attempt", sa.Column("full_exception_trace", sa.Text(), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("index_attempt", "full_exception_trace")

backend/alembic/versions/8aabb57f3b49_restructure_document_indices.py

@@ -0,0 +1,39 @@
+"""Restructure Document Indices
+
+Revision ID: 8aabb57f3b49
+Revises: 5e84129c8be3
+Create Date: 2023-08-18 21:15:57.629515
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "8aabb57f3b49"
+down_revision = "5e84129c8be3"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.drop_table("chunk")
+    op.execute("DROP TYPE IF EXISTS documentstoretype")
+
+
+def downgrade() -> None:
+    op.create_table(
+        "chunk",
+        sa.Column("id", sa.VARCHAR(), autoincrement=False, nullable=False),
+        sa.Column(
+            "document_store_type",
+            postgresql.ENUM("VECTOR", "KEYWORD", name="documentstoretype"),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.Column("document_id", sa.VARCHAR(), autoincrement=False, nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id"], ["document.id"], name="chunk_document_id_fkey"
+        ),
+        sa.PrimaryKeyConstraint("id", "document_store_type", name="chunk_pkey"),
+    )

backend/alembic/versions/8e26726b7683_chat_context_addition.py

@@ -0,0 +1,40 @@
+"""Chat Context Addition
+
+Revision ID: 8e26726b7683
+Revises: 5809c0787398
+Create Date: 2023-09-13 18:34:31.327944
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "8e26726b7683"
+down_revision = "5809c0787398"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "persona",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("system_text", sa.Text(), nullable=True),
+        sa.Column("tools_text", sa.Text(), nullable=True),
+        sa.Column("hint_text", sa.Text(), nullable=True),
+        sa.Column("default_persona", sa.Boolean(), nullable=False),
+        sa.Column("deleted", sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.add_column("chat_message", sa.Column("persona_id", sa.Integer(), nullable=True))
+    op.create_foreign_key(
+        "fk_chat_message_persona_id", "chat_message", "persona", ["persona_id"], ["id"]
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint("fk_chat_message_persona_id", "chat_message", type_="foreignkey")
+    op.drop_column("chat_message", "persona_id")
+    op.drop_table("persona")

backend/alembic/versions/904451035c9b_store_tool_details.py

@@ -0,0 +1,32 @@
+"""Store Tool Details
+
+Revision ID: 904451035c9b
+Revises: 3b25685ff73c
+Create Date: 2023-10-05 12:29:26.620000
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "904451035c9b"
+down_revision = "3b25685ff73c"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "persona",
+        sa.Column("tools", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    )
+    op.drop_column("persona", "tools_text")
+
+
+def downgrade() -> None:
+    op.add_column(
+        "persona",
+        sa.Column("tools_text", sa.TEXT(), autoincrement=False, nullable=True),
+    )
+    op.drop_column("persona", "tools")

backend/alembic/versions/904e5138fffb_tags.py

@@ -0,0 +1,61 @@
+"""Tags
+
+Revision ID: 904e5138fffb
+Revises: 891cd83c87a8
+Create Date: 2024-01-01 10:44:43.733974
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "904e5138fffb"
+down_revision = "891cd83c87a8"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "tag",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("tag_key", sa.String(), nullable=False),
+        sa.Column("tag_value", sa.String(), nullable=False),
+        sa.Column("source", sa.String(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "tag_key", "tag_value", "source", name="_tag_key_value_source_uc"
+        ),
+    )
+    op.create_table(
+        "document__tag",
+        sa.Column("document_id", sa.String(), nullable=False),
+        sa.Column("tag_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id"],
+            ["document.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["tag_id"],
+            ["tag.id"],
+        ),
+        sa.PrimaryKeyConstraint("document_id", "tag_id"),
+    )
+
+    op.add_column(
+        "search_doc",
+        sa.Column(
+            "doc_metadata",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+    )
+    op.execute("UPDATE search_doc SET doc_metadata = '{}' WHERE doc_metadata IS NULL")
+    op.alter_column("search_doc", "doc_metadata", nullable=False)
+
+
+def downgrade() -> None:
+    op.drop_table("document__tag")
+    op.drop_table("tag")
+    op.drop_column("search_doc", "doc_metadata")

backend/alembic/versions/91fd3b470d1a_remove_documentsource_from_tag.py

@@ -0,0 +1,36 @@
+"""Remove DocumentSource from Tag
+
+Revision ID: 91fd3b470d1a
+Revises: 173cae5bba26
+Create Date: 2024-03-21 12:05:23.956734
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from danswer.configs.constants import DocumentSource
+
+# revision identifiers, used by Alembic.
+revision = "91fd3b470d1a"
+down_revision = "173cae5bba26"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "tag",
+        "source",
+        type_=sa.String(length=50),
+        existing_type=sa.Enum(DocumentSource, native_enum=False),
+        existing_nullable=False,
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "tag",
+        "source",
+        type_=sa.Enum(DocumentSource, native_enum=False),
+        existing_type=sa.String(length=50),
+        existing_nullable=False,
+    )

backend/alembic/versions/9d97fecfab7f_added_retrieved_docs_to_query_event.py

@@ -0,0 +1,31 @@
+"""Added retrieved docs to query event
+
+Revision ID: 9d97fecfab7f
+Revises: ffc707a226b4
+Create Date: 2023-10-20 12:22:31.930449
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "9d97fecfab7f"
+down_revision = "ffc707a226b4"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "query_event",
+        sa.Column(
+            "retrieved_document_ids",
+            postgresql.ARRAY(sa.String()),
+            nullable=True,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("query_event", "retrieved_document_ids")

backend/alembic/versions/a570b80a5f20_usergroup_tables.py

@@ -0,0 +1,67 @@
+"""UserGroup tables
+
+Revision ID: a570b80a5f20
+Revises: 904451035c9b
+Create Date: 2023-10-02 12:27:10.265725
+
+"""
+from alembic import op
+import fastapi_users_db_sqlalchemy
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "a570b80a5f20"
+down_revision = "904451035c9b"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "user_group",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("is_up_to_date", sa.Boolean(), nullable=False),
+        sa.Column("is_up_for_deletion", sa.Boolean(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
+    )
+    op.create_table(
+        "user__user_group",
+        sa.Column("user_group_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_group_id"],
+            ["user_group.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("user_group_id", "user_id"),
+    )
+    op.create_table(
+        "user_group__connector_credential_pair",
+        sa.Column("user_group_id", sa.Integer(), nullable=False),
+        sa.Column("cc_pair_id", sa.Integer(), nullable=False),
+        sa.Column("is_current", sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["cc_pair_id"],
+            ["connector_credential_pair.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_group_id"],
+            ["user_group.id"],
+        ),
+        sa.PrimaryKeyConstraint("user_group_id", "cc_pair_id", "is_current"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("user_group__connector_credential_pair")
+    op.drop_table("user__user_group")
+    op.drop_table("user_group")

backend/alembic/versions/ae62505e3acc_add_saml_accounts.py

@@ -0,0 +1,47 @@
+"""Add SAML Accounts
+
+Revision ID: ae62505e3acc
+Revises: 7da543f5672f
+Create Date: 2023-09-26 16:19:30.933183
+
+"""
+import fastapi_users_db_sqlalchemy
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "ae62505e3acc"
+down_revision = "7da543f5672f"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "saml",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=False,
+        ),
+        sa.Column("encrypted_cookie", sa.Text(), nullable=False),
+        sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("encrypted_cookie"),
+        sa.UniqueConstraint("user_id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("saml")

backend/alembic/versions/b082fec533f0_make_last_attempt_status_nullable.py

@@ -0,0 +1,49 @@
+"""Make 'last_attempt_status' nullable
+
+Revision ID: b082fec533f0
+Revises: df0c7ad8a076
+Create Date: 2023-08-06 12:05:47.087325
+
+"""
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "b082fec533f0"
+down_revision = "df0c7ad8a076"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "connector_credential_pair",
+        "last_attempt_status",
+        existing_type=postgresql.ENUM(
+            "NOT_STARTED",
+            "IN_PROGRESS",
+            "SUCCESS",
+            "FAILED",
+            name="indexingstatus",
+        ),
+        nullable=True,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "connector_credential_pair",
+        "last_attempt_status",
+        existing_type=postgresql.ENUM(
+            "NOT_STARTED",
+            "IN_PROGRESS",
+            "SUCCESS",
+            "FAILED",
+            name="indexingstatus",
+        ),
+        nullable=False,
+    )
+    # ### end Alembic commands ###

backend/alembic/versions/b156fa702355_chat_reworked.py

@@ -0,0 +1,520 @@
+"""Chat Reworked
+
+Revision ID: b156fa702355
+Revises: baf71f781b9e
+Create Date: 2023-12-12 00:57:41.823371
+
+"""
+import fastapi_users_db_sqlalchemy
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects.postgresql import ENUM
+from danswer.configs.constants import DocumentSource
+
+# revision identifiers, used by Alembic.
+revision = "b156fa702355"
+down_revision = "baf71f781b9e"
+branch_labels: None = None
+depends_on: None = None
+
+
+searchtype_enum = ENUM(
+    "KEYWORD", "SEMANTIC", "HYBRID", name="searchtype", create_type=True
+)
+recencybiassetting_enum = ENUM(
+    "FAVOR_RECENT",
+    "BASE_DECAY",
+    "NO_DECAY",
+    "AUTO",
+    name="recencybiassetting",
+    create_type=True,
+)
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    searchtype_enum.create(bind)
+    recencybiassetting_enum.create(bind)
+
+    # This is irrecoverable, whatever
+    op.execute("DELETE FROM chat_feedback")
+    op.execute("DELETE FROM document_retrieval_feedback")
+
+    op.create_table(
+        "search_doc",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("document_id", sa.String(), nullable=False),
+        sa.Column("chunk_ind", sa.Integer(), nullable=False),
+        sa.Column("semantic_id", sa.String(), nullable=False),
+        sa.Column("link", sa.String(), nullable=True),
+        sa.Column("blurb", sa.String(), nullable=False),
+        sa.Column("boost", sa.Integer(), nullable=False),
+        sa.Column(
+            "source_type",
+            sa.Enum(DocumentSource, native=False),
+            nullable=False,
+        ),
+        sa.Column("hidden", sa.Boolean(), nullable=False),
+        sa.Column("score", sa.Float(), nullable=False),
+        sa.Column("match_highlights", postgresql.ARRAY(sa.String()), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("primary_owners", postgresql.ARRAY(sa.String()), nullable=True),
+        sa.Column("secondary_owners", postgresql.ARRAY(sa.String()), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "prompt",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("description", sa.String(), nullable=False),
+        sa.Column("system_prompt", sa.Text(), nullable=False),
+        sa.Column("task_prompt", sa.Text(), nullable=False),
+        sa.Column("include_citations", sa.Boolean(), nullable=False),
+        sa.Column("datetime_aware", sa.Boolean(), nullable=False),
+        sa.Column("default_prompt", sa.Boolean(), nullable=False),
+        sa.Column("deleted", sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "persona__prompt",
+        sa.Column("persona_id", sa.Integer(), nullable=False),
+        sa.Column("prompt_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["persona_id"],
+            ["persona.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["prompt_id"],
+            ["prompt.id"],
+        ),
+        sa.PrimaryKeyConstraint("persona_id", "prompt_id"),
+    )
+
+    # Changes to persona first so chat_sessions can have the right persona
+    # The empty persona will be overwritten on server startup
+    op.add_column(
+        "persona",
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "persona",
+        sa.Column(
+            "search_type",
+            searchtype_enum,
+            nullable=True,
+        ),
+    )
+    op.execute("UPDATE persona SET search_type = 'HYBRID'")
+    op.alter_column("persona", "search_type", nullable=False)
+    op.add_column(
+        "persona",
+        sa.Column("llm_relevance_filter", sa.Boolean(), nullable=True),
+    )
+    op.execute("UPDATE persona SET llm_relevance_filter = TRUE")
+    op.alter_column("persona", "llm_relevance_filter", nullable=False)
+    op.add_column(
+        "persona",
+        sa.Column("llm_filter_extraction", sa.Boolean(), nullable=True),
+    )
+    op.execute("UPDATE persona SET llm_filter_extraction = TRUE")
+    op.alter_column("persona", "llm_filter_extraction", nullable=False)
+    op.add_column(
+        "persona",
+        sa.Column(
+            "recency_bias",
+            recencybiassetting_enum,
+            nullable=True,
+        ),
+    )
+    op.execute("UPDATE persona SET recency_bias = 'BASE_DECAY'")
+    op.alter_column("persona", "recency_bias", nullable=False)
+    op.alter_column("persona", "description", existing_type=sa.VARCHAR(), nullable=True)
+    op.execute("UPDATE persona SET description = ''")
+    op.alter_column("persona", "description", nullable=False)
+    op.create_foreign_key("persona__user_fk", "persona", "user", ["user_id"], ["id"])
+    op.drop_column("persona", "datetime_aware")
+    op.drop_column("persona", "tools")
+    op.drop_column("persona", "hint_text")
+    op.drop_column("persona", "apply_llm_relevance_filter")
+    op.drop_column("persona", "retrieval_enabled")
+    op.drop_column("persona", "system_text")
+
+    # Need to create a persona row so fk can work
+    result = bind.execute(sa.text("SELECT 1 FROM persona WHERE id = 0"))
+    exists = result.fetchone()
+    if not exists:
+        op.execute(
+            sa.text(
+                """
+                INSERT INTO persona (
+                    id, user_id, name, description, search_type, num_chunks,
+                    llm_relevance_filter, llm_filter_extraction, recency_bias,
+                    llm_model_version_override, default_persona, deleted
+                ) VALUES (
+                    0, NULL, '', '', 'HYBRID', NULL,
+                    TRUE, TRUE, 'BASE_DECAY', NULL, TRUE, FALSE
+                )
+                """
+            )
+        )
+    delete_statement = sa.text(
+        """
+        DELETE FROM persona
+        WHERE name = 'Danswer' AND default_persona = TRUE AND id != 0
+        """
+    )
+
+    bind.execute(delete_statement)
+
+    op.add_column(
+        "chat_feedback",
+        sa.Column("chat_message_id", sa.Integer(), nullable=False),
+    )
+    op.drop_constraint(
+        "chat_feedback_chat_message_chat_session_id_chat_message_me_fkey",
+        "chat_feedback",
+        type_="foreignkey",
+    )
+    op.drop_column("chat_feedback", "chat_message_edit_number")
+    op.drop_column("chat_feedback", "chat_message_chat_session_id")
+    op.drop_column("chat_feedback", "chat_message_message_number")
+    op.add_column(
+        "chat_message",
+        sa.Column(
+            "id",
+            sa.Integer(),
+            primary_key=True,
+            autoincrement=True,
+            nullable=False,
+            unique=True,
+        ),
+    )
+    op.add_column(
+        "chat_message",
+        sa.Column("parent_message", sa.Integer(), nullable=True),
+    )
+    op.add_column(
+        "chat_message",
+        sa.Column("latest_child_message", sa.Integer(), nullable=True),
+    )
+    op.add_column(
+        "chat_message", sa.Column("rephrased_query", sa.Text(), nullable=True)
+    )
+    op.add_column("chat_message", sa.Column("prompt_id", sa.Integer(), nullable=True))
+    op.add_column(
+        "chat_message",
+        sa.Column("citations", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    )
+    op.add_column("chat_message", sa.Column("error", sa.Text(), nullable=True))
+    op.drop_constraint("fk_chat_message_persona_id", "chat_message", type_="foreignkey")
+    op.create_foreign_key(
+        "chat_message__prompt_fk", "chat_message", "prompt", ["prompt_id"], ["id"]
+    )
+    op.drop_column("chat_message", "parent_edit_number")
+    op.drop_column("chat_message", "persona_id")
+    op.drop_column("chat_message", "reference_docs")
+    op.drop_column("chat_message", "edit_number")
+    op.drop_column("chat_message", "latest")
+    op.drop_column("chat_message", "message_number")
+    op.add_column("chat_session", sa.Column("one_shot", sa.Boolean(), nullable=True))
+    op.execute("UPDATE chat_session SET one_shot = TRUE")
+    op.alter_column("chat_session", "one_shot", nullable=False)
+    op.alter_column(
+        "chat_session",
+        "persona_id",
+        existing_type=sa.INTEGER(),
+        nullable=True,
+    )
+    op.execute("UPDATE chat_session SET persona_id = 0")
+    op.alter_column("chat_session", "persona_id", nullable=False)
+    op.add_column(
+        "document_retrieval_feedback",
+        sa.Column("chat_message_id", sa.Integer(), nullable=False),
+    )
+    op.drop_constraint(
+        "document_retrieval_feedback_qa_event_id_fkey",
+        "document_retrieval_feedback",
+        type_="foreignkey",
+    )
+    op.create_foreign_key(
+        "document_retrieval_feedback__chat_message_fk",
+        "document_retrieval_feedback",
+        "chat_message",
+        ["chat_message_id"],
+        ["id"],
+    )
+    op.drop_column("document_retrieval_feedback", "qa_event_id")
+
+    # Relation table must be created after the other tables are correct
+    op.create_table(
+        "chat_message__search_doc",
+        sa.Column("chat_message_id", sa.Integer(), nullable=False),
+        sa.Column("search_doc_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["chat_message_id"],
+            ["chat_message.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["search_doc_id"],
+            ["search_doc.id"],
+        ),
+        sa.PrimaryKeyConstraint("chat_message_id", "search_doc_id"),
+    )
+
+    # Needs to be created after chat_message id field is added
+    op.create_foreign_key(
+        "chat_feedback__chat_message_fk",
+        "chat_feedback",
+        "chat_message",
+        ["chat_message_id"],
+        ["id"],
+    )
+
+    op.drop_table("query_event")
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "chat_feedback__chat_message_fk", "chat_feedback", type_="foreignkey"
+    )
+    op.drop_constraint(
+        "document_retrieval_feedback__chat_message_fk",
+        "document_retrieval_feedback",
+        type_="foreignkey",
+    )
+    op.drop_constraint("persona__user_fk", "persona", type_="foreignkey")
+    op.drop_constraint("chat_message__prompt_fk", "chat_message", type_="foreignkey")
+    op.drop_constraint(
+        "chat_message__search_doc_chat_message_id_fkey",
+        "chat_message__search_doc",
+        type_="foreignkey",
+    )
+    op.add_column(
+        "persona",
+        sa.Column("system_text", sa.TEXT(), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "persona",
+        sa.Column(
+            "retrieval_enabled",
+            sa.BOOLEAN(),
+            autoincrement=False,
+            nullable=True,
+        ),
+    )
+    op.execute("UPDATE persona SET retrieval_enabled = TRUE")
+    op.alter_column("persona", "retrieval_enabled", nullable=False)
+    op.add_column(
+        "persona",
+        sa.Column(
+            "apply_llm_relevance_filter",
+            sa.BOOLEAN(),
+            autoincrement=False,
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "persona",
+        sa.Column("hint_text", sa.TEXT(), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "persona",
+        sa.Column(
+            "tools",
+            postgresql.JSONB(astext_type=sa.Text()),
+            autoincrement=False,
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "persona",
+        sa.Column("datetime_aware", sa.BOOLEAN(), autoincrement=False, nullable=True),
+    )
+    op.execute("UPDATE persona SET datetime_aware = TRUE")
+    op.alter_column("persona", "datetime_aware", nullable=False)
+    op.alter_column("persona", "description", existing_type=sa.VARCHAR(), nullable=True)
+    op.drop_column("persona", "recency_bias")
+    op.drop_column("persona", "llm_filter_extraction")
+    op.drop_column("persona", "llm_relevance_filter")
+    op.drop_column("persona", "search_type")
+    op.drop_column("persona", "user_id")
+    op.add_column(
+        "document_retrieval_feedback",
+        sa.Column("qa_event_id", sa.INTEGER(), autoincrement=False, nullable=False),
+    )
+    op.drop_column("document_retrieval_feedback", "chat_message_id")
+    op.alter_column(
+        "chat_session", "persona_id", existing_type=sa.INTEGER(), nullable=True
+    )
+    op.drop_column("chat_session", "one_shot")
+    op.add_column(
+        "chat_message",
+        sa.Column(
+            "message_number",
+            sa.INTEGER(),
+            autoincrement=False,
+            nullable=False,
+            primary_key=True,
+        ),
+    )
+    op.add_column(
+        "chat_message",
+        sa.Column("latest", sa.BOOLEAN(), autoincrement=False, nullable=False),
+    )
+    op.add_column(
+        "chat_message",
+        sa.Column(
+            "edit_number",
+            sa.INTEGER(),
+            autoincrement=False,
+            nullable=False,
+            primary_key=True,
+        ),
+    )
+    op.add_column(
+        "chat_message",
+        sa.Column(
+            "reference_docs",
+            postgresql.JSONB(astext_type=sa.Text()),
+            autoincrement=False,
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "chat_message",
+        sa.Column("persona_id", sa.INTEGER(), autoincrement=False, nullable=True),
+    )
+    op.add_column(
+        "chat_message",
+        sa.Column(
+            "parent_edit_number",
+            sa.INTEGER(),
+            autoincrement=False,
+            nullable=True,
+        ),
+    )
+    op.create_foreign_key(
+        "fk_chat_message_persona_id",
+        "chat_message",
+        "persona",
+        ["persona_id"],
+        ["id"],
+    )
+    op.drop_column("chat_message", "error")
+    op.drop_column("chat_message", "citations")
+    op.drop_column("chat_message", "prompt_id")
+    op.drop_column("chat_message", "rephrased_query")
+    op.drop_column("chat_message", "latest_child_message")
+    op.drop_column("chat_message", "parent_message")
+    op.drop_column("chat_message", "id")
+    op.add_column(
+        "chat_feedback",
+        sa.Column(
+            "chat_message_message_number",
+            sa.INTEGER(),
+            autoincrement=False,
+            nullable=False,
+        ),
+    )
+    op.add_column(
+        "chat_feedback",
+        sa.Column(
+            "chat_message_chat_session_id",
+            sa.INTEGER(),
+            autoincrement=False,
+            nullable=False,
+            primary_key=True,
+        ),
+    )
+    op.add_column(
+        "chat_feedback",
+        sa.Column(
+            "chat_message_edit_number",
+            sa.INTEGER(),
+            autoincrement=False,
+            nullable=False,
+        ),
+    )
+    op.drop_column("chat_feedback", "chat_message_id")
+    op.create_table(
+        "query_event",
+        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
+        sa.Column("query", sa.VARCHAR(), autoincrement=False, nullable=False),
+        sa.Column(
+            "selected_search_flow",
+            sa.VARCHAR(),
+            autoincrement=False,
+            nullable=True,
+        ),
+        sa.Column("llm_answer", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("feedback", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True),
+        sa.Column(
+            "time_created",
+            postgresql.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.Column(
+            "retrieved_document_ids",
+            postgresql.ARRAY(sa.VARCHAR()),
+            autoincrement=False,
+            nullable=True,
+        ),
+        sa.Column("chat_session_id", sa.INTEGER(), autoincrement=False, nullable=True),
+        sa.ForeignKeyConstraint(
+            ["chat_session_id"],
+            ["chat_session.id"],
+            name="fk_query_event_chat_session_id",
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"], ["user.id"], name="query_event_user_id_fkey"
+        ),
+        sa.PrimaryKeyConstraint("id", name="query_event_pkey"),
+    )
+    op.drop_table("chat_message__search_doc")
+    op.drop_table("persona__prompt")
+    op.drop_table("prompt")
+    op.drop_table("search_doc")
+    op.create_unique_constraint(
+        "uq_chat_message_combination",
+        "chat_message",
+        ["chat_session_id", "message_number", "edit_number"],
+    )
+    op.create_foreign_key(
+        "chat_feedback_chat_message_chat_session_id_chat_message_me_fkey",
+        "chat_feedback",
+        "chat_message",
+        [
+            "chat_message_chat_session_id",
+            "chat_message_message_number",
+            "chat_message_edit_number",
+        ],
+        ["chat_session_id", "message_number", "edit_number"],
+    )
+    op.create_foreign_key(
+        "document_retrieval_feedback_qa_event_id_fkey",
+        "document_retrieval_feedback",
+        "query_event",
+        ["qa_event_id"],
+        ["id"],
+    )
+
+    op.execute("DROP TYPE IF EXISTS searchtype")
+    op.execute("DROP TYPE IF EXISTS recencybiassetting")
+    op.execute("DROP TYPE IF EXISTS documentsource")

backend/alembic/versions/baf71f781b9e_add_llm_model_version_override_to_.py

@@ -0,0 +1,26 @@
+"""Add llm_model_version_override to Persona
+
+Revision ID: baf71f781b9e
+Revises: 50b683a8295c
+Create Date: 2023-12-06 21:56:50.286158
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "baf71f781b9e"
+down_revision = "50b683a8295c"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "persona",
+        sa.Column("llm_model_version_override", sa.String(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "llm_model_version_override")

backend/alembic/versions/d5645c915d0e_remove_deletion_attempt_table.py

@@ -0,0 +1,73 @@
+"""Remove deletion_attempt table
+
+Revision ID: d5645c915d0e
+Revises: 8e26726b7683
+Create Date: 2023-09-14 15:04:14.444909
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "d5645c915d0e"
+down_revision = "8e26726b7683"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.drop_table("deletion_attempt")
+
+
+def downgrade() -> None:
+    op.create_table(
+        "deletion_attempt",
+        sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False),
+        sa.Column("connector_id", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column("credential_id", sa.INTEGER(), autoincrement=False, nullable=False),
+        sa.Column(
+            "status",
+            postgresql.ENUM(
+                "NOT_STARTED",
+                "IN_PROGRESS",
+                "SUCCESS",
+                "FAILED",
+                name="deletionstatus",
+            ),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.Column(
+            "num_docs_deleted",
+            sa.INTEGER(),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.Column("error_msg", sa.VARCHAR(), autoincrement=False, nullable=True),
+        sa.Column(
+            "time_created",
+            postgresql.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.Column(
+            "time_updated",
+            postgresql.TIMESTAMP(timezone=True),
+            server_default=sa.text("now()"),
+            autoincrement=False,
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["connector_id"],
+            ["connector.id"],
+            name="deletion_attempt_connector_id_fkey",
+        ),
+        sa.ForeignKeyConstraint(
+            ["credential_id"],
+            ["credential.id"],
+            name="deletion_attempt_credential_id_fkey",
+        ),
+        sa.PrimaryKeyConstraint("id", name="deletion_attempt_pkey"),
+    )

backend/alembic/versions/d61e513bef0a_add_total_docs_for_index_attempt.py

@@ -0,0 +1,32 @@
+"""Add Total Docs for Index Attempt
+
+Revision ID: d61e513bef0a
+Revises: 46625e4745d4
+Create Date: 2023-10-27 23:02:43.369964
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "d61e513bef0a"
+down_revision = "46625e4745d4"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "index_attempt",
+        sa.Column("new_docs_indexed", sa.Integer(), nullable=True),
+    )
+    op.alter_column(
+        "index_attempt", "num_docs_indexed", new_column_name="total_docs_indexed"
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "index_attempt", "total_docs_indexed", new_column_name="num_docs_indexed"
+    )
+    op.drop_column("index_attempt", "new_docs_indexed")

backend/alembic/versions/d7111c1238cd_remove_document_ids.py

@@ -0,0 +1,32 @@
+"""Remove Document IDs
+
+Revision ID: d7111c1238cd
+Revises: 465f78d9b7f9
+Create Date: 2023-07-29 15:06:25.126169
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "d7111c1238cd"
+down_revision = "465f78d9b7f9"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.drop_column("index_attempt", "document_ids")
+
+
+def downgrade() -> None:
+    op.add_column(
+        "index_attempt",
+        sa.Column(
+            "document_ids",
+            postgresql.ARRAY(sa.VARCHAR()),
+            autoincrement=False,
+            nullable=True,
+        ),
+    )

backend/alembic/versions/d929f0c1c6af_feedback_feature.py

@@ -0,0 +1,94 @@
+"""Feedback Feature
+
+Revision ID: d929f0c1c6af
+Revises: 8aabb57f3b49
+Create Date: 2023-08-27 13:03:54.274987
+
+"""
+import fastapi_users_db_sqlalchemy
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "d929f0c1c6af"
+down_revision = "8aabb57f3b49"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "query_event",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("query", sa.String(), nullable=False),
+        sa.Column(
+            "selected_search_flow",
+            sa.Enum("KEYWORD", "SEMANTIC", name="searchtype", native_enum=False),
+            nullable=True,
+        ),
+        sa.Column("llm_answer", sa.String(), nullable=True),
+        sa.Column(
+            "feedback",
+            sa.Enum("LIKE", "DISLIKE", name="qafeedbacktype", native_enum=False),
+            nullable=True,
+        ),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=True,
+        ),
+        sa.Column(
+            "time_created",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "document_retrieval_feedback",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("qa_event_id", sa.Integer(), nullable=False),
+        sa.Column("document_id", sa.String(), nullable=False),
+        sa.Column("document_rank", sa.Integer(), nullable=False),
+        sa.Column("clicked", sa.Boolean(), nullable=False),
+        sa.Column(
+            "feedback",
+            sa.Enum(
+                "ENDORSE",
+                "REJECT",
+                "HIDE",
+                "UNHIDE",
+                name="searchfeedbacktype",
+                native_enum=False,
+            ),
+            nullable=True,
+        ),
+        sa.ForeignKeyConstraint(
+            ["document_id"],
+            ["document.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["qa_event_id"],
+            ["query_event.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.add_column("document", sa.Column("boost", sa.Integer(), nullable=False))
+    op.add_column("document", sa.Column("hidden", sa.Boolean(), nullable=False))
+    op.add_column("document", sa.Column("semantic_id", sa.String(), nullable=False))
+    op.add_column("document", sa.Column("link", sa.String(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("document", "link")
+    op.drop_column("document", "semantic_id")
+    op.drop_column("document", "hidden")
+    op.drop_column("document", "boost")
+    op.drop_table("document_retrieval_feedback")
+    op.drop_table("query_event")

backend/alembic/versions/dba7f71618f5_danswer_custom_tool_flow.py

@@ -0,0 +1,29 @@
+"""Danswer Custom Tool Flow
+
+Revision ID: dba7f71618f5
+Revises: d5645c915d0e
+Create Date: 2023-09-18 15:18:37.370972
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "dba7f71618f5"
+down_revision = "d5645c915d0e"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "persona",
+        sa.Column("retrieval_enabled", sa.Boolean(), nullable=True),
+    )
+    op.execute("UPDATE persona SET retrieval_enabled = true")
+    op.alter_column("persona", "retrieval_enabled", nullable=False)
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "retrieval_enabled")

backend/alembic/versions/dbaa756c2ccf_embedding_models.py

@@ -0,0 +1,139 @@
+"""Embedding Models
+
+Revision ID: dbaa756c2ccf
+Revises: 7f726bad5367
+Create Date: 2024-01-25 17:12:31.813160
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import table, column, String, Integer, Boolean
+
+from danswer.db.embedding_model import (
+    get_new_default_embedding_model,
+    get_old_default_embedding_model,
+    user_has_overridden_embedding_model,
+)
+from danswer.db.models import IndexModelStatus
+
+# revision identifiers, used by Alembic.
+revision = "dbaa756c2ccf"
+down_revision = "7f726bad5367"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "embedding_model",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("model_name", sa.String(), nullable=False),
+        sa.Column("model_dim", sa.Integer(), nullable=False),
+        sa.Column("normalize", sa.Boolean(), nullable=False),
+        sa.Column("query_prefix", sa.String(), nullable=False),
+        sa.Column("passage_prefix", sa.String(), nullable=False),
+        sa.Column("index_name", sa.String(), nullable=False),
+        sa.Column(
+            "status",
+            sa.Enum(IndexModelStatus, native=False),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # since all index attempts must be associated with an embedding model,
+    # need to put something in here to avoid nulls. On server startup,
+    # this value will be overriden
+    EmbeddingModel = table(
+        "embedding_model",
+        column("id", Integer),
+        column("model_name", String),
+        column("model_dim", Integer),
+        column("normalize", Boolean),
+        column("query_prefix", String),
+        column("passage_prefix", String),
+        column("index_name", String),
+        column(
+            "status", sa.Enum(IndexModelStatus, name="indexmodelstatus", native=False)
+        ),
+    )
+    # insert an embedding model row that corresponds to the embedding model
+    # the user selected via env variables before this change. This is needed since
+    # all index_attempts must be associated with an embedding model, so without this
+    # we will run into violations of non-null contraints
+    old_embedding_model = get_old_default_embedding_model()
+    op.bulk_insert(
+        EmbeddingModel,
+        [
+            {
+                "model_name": old_embedding_model.model_name,
+                "model_dim": old_embedding_model.model_dim,
+                "normalize": old_embedding_model.normalize,
+                "query_prefix": old_embedding_model.query_prefix,
+                "passage_prefix": old_embedding_model.passage_prefix,
+                "index_name": old_embedding_model.index_name,
+                "status": old_embedding_model.status,
+            }
+        ],
+    )
+    # if the user has not overridden the default embedding model via env variables,
+    # insert the new default model into the database to auto-upgrade them
+    if not user_has_overridden_embedding_model():
+        new_embedding_model = get_new_default_embedding_model(is_present=False)
+        op.bulk_insert(
+            EmbeddingModel,
+            [
+                {
+                    "model_name": new_embedding_model.model_name,
+                    "model_dim": new_embedding_model.model_dim,
+                    "normalize": new_embedding_model.normalize,
+                    "query_prefix": new_embedding_model.query_prefix,
+                    "passage_prefix": new_embedding_model.passage_prefix,
+                    "index_name": new_embedding_model.index_name,
+                    "status": IndexModelStatus.FUTURE,
+                }
+            ],
+        )
+
+    op.add_column(
+        "index_attempt",
+        sa.Column("embedding_model_id", sa.Integer(), nullable=True),
+    )
+    op.execute(
+        "UPDATE index_attempt SET embedding_model_id=1 WHERE embedding_model_id IS NULL"
+    )
+    op.alter_column(
+        "index_attempt",
+        "embedding_model_id",
+        existing_type=sa.Integer(),
+        nullable=False,
+    )
+    op.create_foreign_key(
+        "index_attempt__embedding_model_fk",
+        "index_attempt",
+        "embedding_model",
+        ["embedding_model_id"],
+        ["id"],
+    )
+    op.create_index(
+        "ix_embedding_model_present_unique",
+        "embedding_model",
+        ["status"],
+        unique=True,
+        postgresql_where=sa.text("status = 'PRESENT'"),
+    )
+    op.create_index(
+        "ix_embedding_model_future_unique",
+        "embedding_model",
+        ["status"],
+        unique=True,
+        postgresql_where=sa.text("status = 'FUTURE'"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "index_attempt__embedding_model_fk", "index_attempt", type_="foreignkey"
+    )
+    op.drop_column("index_attempt", "embedding_model_id")
+    op.drop_table("embedding_model")
+    op.execute("DROP TYPE indexmodelstatus;")

backend/alembic/versions/df0c7ad8a076_added_deletion_attempt_table.py

@@ -0,0 +1,111 @@
+"""Added deletion_attempt table
+
+Revision ID: df0c7ad8a076
+Revises: d7111c1238cd
+Create Date: 2023-08-05 13:35:39.609619
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "df0c7ad8a076"
+down_revision = "d7111c1238cd"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "document",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "chunk",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column(
+            "document_store_type",
+            sa.Enum(
+                "VECTOR",
+                "KEYWORD",
+                name="documentstoretype",
+                native_enum=False,
+            ),
+            nullable=False,
+        ),
+        sa.Column("document_id", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id"],
+            ["document.id"],
+        ),
+        sa.PrimaryKeyConstraint("id", "document_store_type"),
+    )
+    op.create_table(
+        "deletion_attempt",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("connector_id", sa.Integer(), nullable=False),
+        sa.Column("credential_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "status",
+            sa.Enum(
+                "NOT_STARTED",
+                "IN_PROGRESS",
+                "SUCCESS",
+                "FAILED",
+                name="deletionstatus",
+                native_enum=False,
+            ),
+            nullable=False,
+        ),
+        sa.Column("num_docs_deleted", sa.Integer(), nullable=False),
+        sa.Column("error_msg", sa.String(), nullable=True),
+        sa.Column(
+            "time_created",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "time_updated",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["connector_id"],
+            ["connector.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["credential_id"],
+            ["credential.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "document_by_connector_credential_pair",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column("connector_id", sa.Integer(), nullable=False),
+        sa.Column("credential_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["connector_id"],
+            ["connector.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["credential_id"],
+            ["credential.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["id"],
+            ["document.id"],
+        ),
+        sa.PrimaryKeyConstraint("id", "connector_id", "credential_id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("document_by_connector_credential_pair")
+    op.drop_table("deletion_attempt")
+    op.drop_table("chunk")
+    op.drop_table("document")

backend/alembic/versions/e0a68a81d434_add_chat_feedback.py

@@ -0,0 +1,44 @@
+"""Add Chat Feedback
+
+Revision ID: e0a68a81d434
+Revises: ae62505e3acc
+Create Date: 2023-10-04 20:22:33.380286
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "e0a68a81d434"
+down_revision = "ae62505e3acc"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "chat_feedback",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("chat_message_chat_session_id", sa.Integer(), nullable=False),
+        sa.Column("chat_message_message_number", sa.Integer(), nullable=False),
+        sa.Column("chat_message_edit_number", sa.Integer(), nullable=False),
+        sa.Column("is_positive", sa.Boolean(), nullable=True),
+        sa.Column("feedback_text", sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(
+            [
+                "chat_message_chat_session_id",
+                "chat_message_message_number",
+                "chat_message_edit_number",
+            ],
+            [
+                "chat_message.chat_session_id",
+                "chat_message.message_number",
+                "chat_message.edit_number",
+            ],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("chat_feedback")

backend/alembic/versions/e50154680a5c_no_source_enum.py

@@ -0,0 +1,38 @@
+"""No Source Enum
+
+Revision ID: e50154680a5c
+Revises: fcd135795f21
+Create Date: 2024-03-14 18:06:08.523106
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+from danswer.configs.constants import DocumentSource
+
+# revision identifiers, used by Alembic.
+revision = "e50154680a5c"
+down_revision = "fcd135795f21"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "search_doc",
+        "source_type",
+        type_=sa.String(length=50),
+        existing_type=sa.Enum(DocumentSource, native_enum=False),
+        existing_nullable=False,
+    )
+    op.execute("DROP TYPE IF EXISTS documentsource")
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "search_doc",
+        "source_type",
+        type_=sa.Enum(DocumentSource, native_enum=False),
+        existing_type=sa.String(length=50),
+        existing_nullable=False,
+    )

backend/alembic/versions/e6a4bbc13fe4_add_index_for_retrieving_latest_index_.py

@@ -0,0 +1,31 @@
+"""Add index for retrieving latest index_attempt
+
+Revision ID: e6a4bbc13fe4
+Revises: b082fec533f0
+Create Date: 2023-08-10 12:37:23.335471
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "e6a4bbc13fe4"
+down_revision = "b082fec533f0"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_index(
+        op.f("ix_index_attempt_latest_for_connector_credential_pair"),
+        "index_attempt",
+        ["connector_id", "credential_id", "time_created"],
+        unique=False,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        op.f("ix_index_attempt_latest_for_connector_credential_pair"),
+        table_name="index_attempt",
+    )

backend/alembic/versions/e86866a9c78a_add_persona_to_chat_session.py

@@ -0,0 +1,27 @@
+"""Add persona to chat_session
+
+Revision ID: e86866a9c78a
+Revises: 80696cf850ae
+Create Date: 2023-11-26 02:51:47.657357
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "e86866a9c78a"
+down_revision = "80696cf850ae"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column("chat_session", sa.Column("persona_id", sa.Integer(), nullable=True))
+    op.create_foreign_key(
+        "fk_chat_session_persona_id", "chat_session", "persona", ["persona_id"], ["id"]
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint("fk_chat_session_persona_id", "chat_session", type_="foreignkey")
+    op.drop_column("chat_session", "persona_id")

backend/alembic/versions/e91df4e935ef_private_personas_documentsets.py

@@ -0,0 +1,118 @@
+"""Private Personas DocumentSets
+
+Revision ID: e91df4e935ef
+Revises: 91fd3b470d1a
+Create Date: 2024-03-17 11:47:24.675881
+
+"""
+import fastapi_users_db_sqlalchemy
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "e91df4e935ef"
+down_revision = "91fd3b470d1a"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "document_set__user",
+        sa.Column("document_set_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["document_set_id"],
+            ["document_set.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("document_set_id", "user_id"),
+    )
+    op.create_table(
+        "persona__user",
+        sa.Column("persona_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "user_id",
+            fastapi_users_db_sqlalchemy.generics.GUID(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["persona_id"],
+            ["persona.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user.id"],
+        ),
+        sa.PrimaryKeyConstraint("persona_id", "user_id"),
+    )
+    op.create_table(
+        "document_set__user_group",
+        sa.Column("document_set_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "user_group_id",
+            sa.Integer(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["document_set_id"],
+            ["document_set.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_group_id"],
+            ["user_group.id"],
+        ),
+        sa.PrimaryKeyConstraint("document_set_id", "user_group_id"),
+    )
+    op.create_table(
+        "persona__user_group",
+        sa.Column("persona_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "user_group_id",
+            sa.Integer(),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["persona_id"],
+            ["persona.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_group_id"],
+            ["user_group.id"],
+        ),
+        sa.PrimaryKeyConstraint("persona_id", "user_group_id"),
+    )
+
+    op.add_column(
+        "document_set",
+        sa.Column("is_public", sa.Boolean(), nullable=True),
+    )
+    # fill in is_public for existing rows
+    op.execute("UPDATE document_set SET is_public = true WHERE is_public IS NULL")
+    op.alter_column("document_set", "is_public", nullable=False)
+
+    op.add_column(
+        "persona",
+        sa.Column("is_public", sa.Boolean(), nullable=True),
+    )
+    # fill in is_public for existing rows
+    op.execute("UPDATE persona SET is_public = true WHERE is_public IS NULL")
+    op.alter_column("persona", "is_public", nullable=False)
+
+
+def downgrade() -> None:
+    op.drop_column("persona", "is_public")
+
+    op.drop_column("document_set", "is_public")
+
+    op.drop_table("persona__user")
+    op.drop_table("document_set__user")
+    op.drop_table("persona__user_group")
+    op.drop_table("document_set__user_group")

backend/alembic/versions/ec3ec2eabf7b_index_from_beginning.py

@@ -0,0 +1,27 @@
+"""Index From Beginning
+
+Revision ID: ec3ec2eabf7b
+Revises: dbaa756c2ccf
+Create Date: 2024-02-06 22:03:28.098158
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "ec3ec2eabf7b"
+down_revision = "dbaa756c2ccf"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "index_attempt", sa.Column("from_beginning", sa.Boolean(), nullable=True)
+    )
+    op.execute("UPDATE index_attempt SET from_beginning = False")
+    op.alter_column("index_attempt", "from_beginning", nullable=False)
+
+
+def downgrade() -> None:
+    op.drop_column("index_attempt", "from_beginning")

backend/alembic/versions/ecab2b3f1a3b_add_overrides_to_the_chat_session.py

@@ -0,0 +1,40 @@
+"""Add overrides to the chat session
+
+Revision ID: ecab2b3f1a3b
+Revises: 38eda64af7fe
+Create Date: 2024-04-01 19:08:21.359102
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "ecab2b3f1a3b"
+down_revision = "38eda64af7fe"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_session",
+        sa.Column(
+            "llm_override",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+    )
+    op.add_column(
+        "chat_session",
+        sa.Column(
+            "prompt_override",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("chat_session", "prompt_override")
+    op.drop_column("chat_session", "llm_override")

backend/alembic/versions/ef7da92f7213_add_files_to_chatmessage.py

@@ -0,0 +1,27 @@
+"""Add files to ChatMessage
+
+Revision ID: ef7da92f7213
+Revises: 401c1ac29467
+Create Date: 2024-04-28 16:59:33.199153
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "ef7da92f7213"
+down_revision = "401c1ac29467"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_message",
+        sa.Column("files", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("chat_message", "files")

backend/alembic/versions/f1c6478c3fd8_add_pre_defined_feedback.py

@@ -0,0 +1,25 @@
+"""Add pre-defined feedback
+
+Revision ID: f1c6478c3fd8
+Revises: 643a84a42a33
+Create Date: 2024-05-09 18:11:49.210667
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "f1c6478c3fd8"
+down_revision = "643a84a42a33"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "chat_feedback",
+        sa.Column("predefined_feedback", sa.String(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("chat_feedback", "predefined_feedback")

backend/alembic/versions/fad14119fb92_delete_tags_with_wrong_enum.py

@@ -0,0 +1,39 @@
+"""Delete Tags with wrong Enum
+
+Revision ID: fad14119fb92
+Revises: 72bdc9929a46
+Create Date: 2024-04-25 17:05:09.695703
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "fad14119fb92"
+down_revision = "72bdc9929a46"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    # Some documents may lose their tags but this is the only way as the enum
+    # mapping may have changed since tag switched to string (it will be reindexed anyway)
+    op.execute(
+        """
+        DELETE FROM document__tag
+        WHERE tag_id IN (
+            SELECT id FROM tag
+            WHERE source ~ '^[0-9]+$'
+        )
+        """
+    )
+
+    op.execute(
+        """
+        DELETE FROM tag
+        WHERE source ~ '^[0-9]+$'
+        """
+    )
+
+
+def downgrade() -> None:
+    pass

backend/alembic/versions/fcd135795f21_add_slack_bot_display_type.py

@@ -0,0 +1,39 @@
+"""Add slack bot display type
+
+Revision ID: fcd135795f21
+Revises: 0a2b51deb0b8
+Create Date: 2024-03-04 17:03:27.116284
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "fcd135795f21"
+down_revision = "0a2b51deb0b8"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "slack_bot_config",
+        sa.Column(
+            "response_type",
+            sa.Enum(
+                "QUOTES",
+                "CITATIONS",
+                name="slackbotresponsetype",
+                native_enum=False,
+            ),
+            nullable=True,
+        ),
+    )
+    op.execute(
+        "UPDATE slack_bot_config SET response_type = 'QUOTES' WHERE response_type IS NULL"
+    )
+    op.alter_column("slack_bot_config", "response_type", nullable=False)
+
+
+def downgrade() -> None:
+    op.drop_column("slack_bot_config", "response_type")

backend/alembic/versions/febe9eaa0644_add_document_set_persona_relationship_.py

@@ -0,0 +1,37 @@
+"""Add document_set / persona relationship table
+
+Revision ID: febe9eaa0644
+Revises: 57b53544726e
+Create Date: 2023-09-24 13:06:24.018610
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "febe9eaa0644"
+down_revision = "57b53544726e"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "persona__document_set",
+        sa.Column("persona_id", sa.Integer(), nullable=False),
+        sa.Column("document_set_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_set_id"],
+            ["document_set.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["persona_id"],
+            ["persona.id"],
+        ),
+        sa.PrimaryKeyConstraint("persona_id", "document_set_id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("persona__document_set")

backend/alembic/versions/ffc707a226b4_basic_document_metadata.py

@@ -0,0 +1,37 @@
+"""Basic Document Metadata
+
+Revision ID: ffc707a226b4
+Revises: 30c1d5744104
+Create Date: 2023-10-18 16:52:25.967592
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "ffc707a226b4"
+down_revision = "30c1d5744104"
+branch_labels: None = None
+depends_on: None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "document",
+        sa.Column("doc_updated_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.add_column(
+        "document",
+        sa.Column("primary_owners", postgresql.ARRAY(sa.String()), nullable=True),
+    )
+    op.add_column(
+        "document",
+        sa.Column("secondary_owners", postgresql.ARRAY(sa.String()), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("document", "secondary_owners")
+    op.drop_column("document", "primary_owners")
+    op.drop_column("document", "doc_updated_at")

backend/danswer/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+__version__ = os.environ.get("DANSWER_VERSION", "") or "0.3-dev"

backend/danswer/access/access.py

@@ -0,0 +1,62 @@
+from sqlalchemy.orm import Session
+
+from danswer.access.models import DocumentAccess
+from danswer.configs.constants import PUBLIC_DOC_PAT
+from danswer.db.document import get_acccess_info_for_documents
+from danswer.db.models import User
+from danswer.server.documents.models import ConnectorCredentialPairIdentifier
+from danswer.utils.variable_functionality import fetch_versioned_implementation
+
+
+def _get_access_for_documents(
+    document_ids: list[str],
+    db_session: Session,
+    cc_pair_to_delete: ConnectorCredentialPairIdentifier | None = None,
+) -> dict[str, DocumentAccess]:
+    document_access_info = get_acccess_info_for_documents(
+        db_session=db_session,
+        document_ids=document_ids,
+        cc_pair_to_delete=cc_pair_to_delete,
+    )
+    return {
+        document_id: DocumentAccess.build(user_ids, is_public)
+        for document_id, user_ids, is_public in document_access_info
+    }
+
+
+def get_access_for_documents(
+    document_ids: list[str],
+    db_session: Session,
+    cc_pair_to_delete: ConnectorCredentialPairIdentifier | None = None,
+) -> dict[str, DocumentAccess]:
+    """Fetches all access information for the given documents."""
+    versioned_get_access_for_documents_fn = fetch_versioned_implementation(
+        "danswer.access.access", "_get_access_for_documents"
+    )
+    return versioned_get_access_for_documents_fn(
+        document_ids, db_session, cc_pair_to_delete
+    )  # type: ignore
+
+
+def prefix_user(user_id: str) -> str:
+    """Prefixes a user ID to eliminate collision with group names.
+    This assumes that groups are prefixed with a different prefix."""
+    return f"user_id:{user_id}"
+
+
+def _get_acl_for_user(user: User | None, db_session: Session) -> set[str]:
+    """Returns a list of ACL entries that the user has access to. This is meant to be
+    used downstream to filter out documents that the user does not have access to. The
+    user should have access to a document if at least one entry in the document's ACL
+    matches one entry in the returned set.
+    """
+    if user:
+        return {prefix_user(str(user.id)), PUBLIC_DOC_PAT}
+    return {PUBLIC_DOC_PAT}
+
+
+def get_acl_for_user(user: User | None, db_session: Session | None = None) -> set[str]:
+    versioned_acl_for_user_fn = fetch_versioned_implementation(
+        "danswer.access.access", "_get_acl_for_user"
+    )
+    return versioned_acl_for_user_fn(user, db_session)  # type: ignore