1
0
forked from github/onyx

Compare commits

...

12 Commits

Author SHA1 Message Date
Devin AI
19a135f552 style: fix prettier formatting in validation schema reducer
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 02:43:35 +00:00
Devin AI
c5bfbe0472 style: fix prettier formatting in validation schema reducer
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 02:36:06 +00:00
Devin AI
85784b0a2d Fix Prettier formatting in connectors.tsx
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 02:27:15 +00:00
Devin AI
4216264ece style: update prettier formatting in reduce functions
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 02:23:07 +00:00
Devin AI
778ab515fd style: fix prettier formatting in connectors.tsx
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 02:17:38 +00:00
Devin AI
971366ff06 style: fix prettier formatting in Slack connector config
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 02:13:29 +00:00
Devin AI
e617445da6 fix: format description field in Slack connector config
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 02:05:26 +00:00
Devin AI
a378c30b67 feat(slack): add ignore_private_channels checkbox to Slack connector UI
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 02:02:47 +00:00
Devin AI
dfda56ee41 style: apply black formatting
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 01:22:14 +00:00
Devin AI
dbdfcf5391 style: fix black formatting
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 01:21:25 +00:00
Devin AI
b63a0f8ea2 style: format code with black
Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 01:17:02 +00:00
Devin AI
918a4fa560 Add ignore_private_channels option to Slack connector
- Add ignore_private_channels parameter to SlackPollConnector
- Update channel fetching logic to respect the new parameter
- Update doc_sync.py to handle private channel filtering
- Add comprehensive documentation

Co-Authored-By: Chris Weaver <chris@onyx.app>
2025-01-27 01:13:45 +00:00
3 changed files with 65 additions and 7 deletions

View File

@@ -53,7 +53,19 @@ def _fetch_channel_permissions(
slack_client: WebClient,
workspace_permissions: ExternalAccess,
user_id_to_email_map: dict[str, str],
ignore_private_channels: bool = False,
) -> dict[str, ExternalAccess]:
"""Fetch permissions for all channels.
Args:
slack_client: The Slack WebClient instance.
workspace_permissions: Base permissions for the workspace.
user_id_to_email_map: Mapping of Slack user IDs to email addresses.
ignore_private_channels: If True, skip fetching private channel permissions.
Returns:
A dictionary mapping channel IDs to their ExternalAccess permissions.
"""
channel_permissions = {}
public_channels = get_channels(
client=slack_client,
@@ -66,11 +78,12 @@ def _fetch_channel_permissions(
for channel_id in public_channel_ids:
channel_permissions[channel_id] = workspace_permissions
private_channels = get_channels(
client=slack_client,
get_public=False,
get_private=True,
)
if not ignore_private_channels: # Only fetch private channels if not ignored
private_channels = get_channels(
client=slack_client,
get_public=False,
get_private=True,
)
private_channel_ids = [
channel["id"] for channel in private_channels if "id" in channel
]
@@ -132,10 +145,15 @@ def slack_doc_sync(
workspace_permissions = _fetch_workspace_permissions(
user_id_to_email_map=user_id_to_email_map,
)
# Pass through ignore_private_channels from connector config
ignore_private_channels = cc_pair.connector.connector_specific_config.get(
"ignore_private_channels", False
)
channel_permissions = _fetch_channel_permissions(
slack_client=slack_client,
workspace_permissions=workspace_permissions,
user_id_to_email_map=user_id_to_email_map,
ignore_private_channels=ignore_private_channels,
)
document_external_accesses = []

View File

@@ -267,6 +267,7 @@ def _get_all_docs(
oldest: str | None = None,
latest: str | None = None,
msg_filter_func: Callable[[MessageType], bool] = default_msg_filter,
ignore_private_channels: bool = False, # Add parameter to function
) -> Generator[Document, None, None]:
"""Get all documents in the workspace, channel by channel"""
slack_cleaner = SlackTextCleaner(client=client)
@@ -274,7 +275,10 @@ def _get_all_docs(
# Cache to prevent refetching via API since users
user_cache: dict[str, BasicExpertInfo | None] = {}
all_channels = get_channels(client)
all_channels = get_channels(
client=client,
get_private=not ignore_private_channels, # Skip private channels if configured
)
filtered_channels = filter_channels(
all_channels, channels, channel_name_regex_enabled
)
@@ -325,6 +329,7 @@ def _get_all_doc_ids(
channels: list[str] | None = None,
channel_name_regex_enabled: bool = False,
msg_filter_func: Callable[[MessageType], bool] = default_msg_filter,
ignore_private_channels: bool = False, # Add parameter to function
) -> GenerateSlimDocumentOutput:
"""
Get all document ids in the workspace, channel by channel
@@ -332,7 +337,10 @@ def _get_all_doc_ids(
This makes it an order of magnitude faster than get_all_docs
"""
all_channels = get_channels(client)
all_channels = get_channels(
client=client,
get_private=not ignore_private_channels, # Skip private channels if configured
)
filtered_channels = filter_channels(
all_channels, channels, channel_name_regex_enabled
)
@@ -368,6 +376,11 @@ def _get_all_doc_ids(
class SlackPollConnector(PollConnector, SlimConnector):
"""A connector for polling Slack channels and retrieving messages.
This connector allows configuring which channels to index and how to handle private channels.
"""
def __init__(
self,
channels: list[str] | None = None,
@@ -375,15 +388,30 @@ class SlackPollConnector(PollConnector, SlimConnector):
# regexes, and will only index channels that fully match the regexes
channel_regex_enabled: bool = False,
batch_size: int = INDEX_BATCH_SIZE,
# if True, private channels will be ignored during indexing
ignore_private_channels: bool = False,
) -> None:
"""Initialize the Slack connector.
Args:
channels: Optional list of channel names to index. If None, all accessible channels are indexed.
channel_regex_enabled: If True, treat channel names as regex patterns.
batch_size: Number of documents to process in each batch.
ignore_private_channels: If True, skip indexing private channels entirely.
"""
self.channels = channels
self.channel_regex_enabled = channel_regex_enabled
self.batch_size = batch_size
self.ignore_private_channels = ignore_private_channels
self.client: WebClient | None = None
def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
bot_token = credentials["slack_bot_token"]
self.client = WebClient(token=bot_token)
# Update ignore_private_channels from credentials if provided
self.ignore_private_channels = bool(
credentials.get("ignore_private_channels", self.ignore_private_channels)
)
return None
def retrieve_all_slim_documents(
@@ -398,6 +426,7 @@ class SlackPollConnector(PollConnector, SlimConnector):
client=self.client,
channels=self.channels,
channel_name_regex_enabled=self.channel_regex_enabled,
ignore_private_channels=self.ignore_private_channels,
)
def poll_source(
@@ -416,6 +445,7 @@ class SlackPollConnector(PollConnector, SlimConnector):
# retention
oldest=str(start) if start else None,
latest=str(end),
ignore_private_channels=self.ignore_private_channels,
):
documents.append(document)
if len(documents) >= self.batch_size:

View File

@@ -568,6 +568,16 @@ Hint: Use the singular form of the object name (e.g., 'Opportunity' instead of '
For example, specifying .*-support.* as a "channel" will cause the connector to include any channels with "-support" in the name.`,
optional: true,
},
{
type: "checkbox",
query: "Ignore private channels?",
label: "Ignore Private Channels",
name: "ignore_private_channels",
description:
"If checked, private Slack channels will be ignored during indexing.",
optional: true,
default: false,
},
],
},
slab: {