Bulk Deleting Microsoft Sentinel Threat Intelligence Indicators by Source

Microsoft hasn't implemented a way to bulk delete indicators into Sentinel which might be an issue if you have conneted to a very noisy feed. This code runs in a Jupyter notebook but if you want to run it standalone you will need to change or remove tqdm (progress bar). The progress bar doesn't show actualy progress, it just moves to indicate "something" is happening.

I am not using the "nextLink" to iterate as it doesn't seem to function correctly using the the "queryIndicators" API endpoint. It's not an issue here as each query should return results that aren't deleted.

The code "works" but there are a number of things that could be improved or fixed. I've succesfully used but I'd recommend understanding Python. Use at your own risk.

import pendulum
from msal import ConfidentialClientApplication
from tqdm.notebook import tqdm
import httpx
import random
import asyncio
import json
import urllib

client_secret = ""
client_id = ""
tenant_id = ""
api_version = "2024-01-01-preview"
subscription = ""
resource_group = ""
workspace = ""
sources_to_delete = []

api_target = f"https://management.azure.com/subscriptions/{subscription}/resourceGroups/{resource_group}/providers/Microsoft.OperationalInsights/workspaces/{workspace}/providers/Microsoft.SecurityInsights/threatIntelligence/main/queryIndicators?api-version={api_version}"
json_body = {
    "sources": sources_to_delete,
    "pageSize": 100,
}
token_issued = pendulum.now()
token_expires = token_issued.subtract(minutes=1)
issued_token = None

app = ConfidentialClientApplication(
    client_id, client_secret, authority=f"https://login.microsoftonline.com/{tenant_id}"
)


def get_token():
    global issued_token
    global token_expires

    if token_expires <= pendulum.now().add(minutes=5):
        print("Refreshing token.")
        issued_token = app.acquire_token_for_client(
            scopes=["https://management.azure.com/.default"]
        )
        token_expires = pendulum.now().add(seconds=issued_token["expires_in"])
        print(f"New expiry: {token_expires.to_iso8601_string()}")
    return issued_token

token = get_token()


def random_bar(bar):
    bar.n = 0 + random.randint(0, 100)
    bar.refresh()
    bar.set_description(random.choice(["Working hard", "Hardly working"]))
headers = {"Authorization": f"Bearer {get_token()['access_token']}"}


bar = tqdm(total=100)


async def delete_indicator(indicator):

    async with httpx.AsyncClient() as client:
        headers = {"Authorization": f"Bearer {get_token()['access_token']}"}

        r_delete = await client.delete(
            f"https://management.azure.com{indicator}?api-version={api_version}",
            headers=headers,
            # params=params,
        )
        r_delete.raise_for_status()


async def gather_deletes(indicators):
    current_indicators = set()
    random_bar(bar)
    bar.set_description(f"Seen {len(seen_indicators)}")
    for indicator in indicators["value"]:
        if indicator["id"] in seen_indicators:
            continue
        seen_indicators.add(indicator["id"])
        current_indicators.add(indicator["id"])

    async with asyncio.TaskGroup() as tg:
        for indicator in current_indicators:
            tg.create_task(delete_indicator(indicator))


async def do_deletes(payload):
    await gather_deletes(payload)

    if "nextLink" in payload:

        r = httpx.post(api_target, headers=headers, json=json_body)

        await do_deletes(r.json())


r = httpx.post(api_target, headers=headers, json=json_body)

await do_deletes(r.json())
bar.close()

A correction to a previous note, indicators are not deleted or deactivated in the 'ThreatIntelligenceIndicator'. You should ensure your retention period on the 'ThreatIntelligenceIndicator' table is 30 days. Once an indicator is removed, it will not updated in this table and be removed after this period. Existing indicators should continue to generate new entries in this table