Skip to content

Deprecate Get-PnPSearchCrawlLog cmdlet #5229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wobba wants to merge 1 commit into
base: dev
Choose a base branch
from
+6 −188
Draft

Deprecate Get-PnPSearchCrawlLog cmdlet #5229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).

### Removed
- Removed `-RemoveExisting` parameter from `Add-PnPAzureADGroupMember`, `Add-PnPAzureADGroupOwner`, `Add-PnPMicrosoft365GroupMember` and `Add-PnPMicrosoft365GroupOwner` cmdlets. It was never really implemented and without function. [#5153](https://github.com/pnp/powershell/pull/5153)
- Deprecated `Get-PnPSearchCrawlLog` cmdlet as the underlying API has been deprecated by Microsoft and no longer returns results. The cmdlet now throws a `NotSupportedException` and will be removed in a future version.

### Contributors

Expand Down
3 changes: 3 additions & 0 deletions documentation/Get-PnPSearchCrawlLog.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ online version: https://pnp.github.io/powershell/cmdlets/Get-PnPSearchCrawlLog.h

# Get-PnPSearchCrawlLog

> [!CAUTION]
> **This cmdlet has been deprecated.** The underlying API has been deprecated by Microsoft and no longer returns results. This cmdlet will be removed in a future version of PnP PowerShell.
## SYNOPSIS
Returns entries from the SharePoint search crawl log. Make sure you are granted access to the crawl log via the SharePoint search admin center at https://<tenant>-admin.sharepoint.com/_layouts/15/searchadmin/crawllogreadpermission.aspx in order to run this cmdlet.

Expand Down
42 changes: 0 additions & 42 deletions resources/predictor/PnP.PowerShell.Suggestions.nightly.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4355,48 +4355,6 @@
"Id": 726,
"Rank": 8
},
{
"CommandName": "Get-PnPSearchCrawlLog",
"Command": "Get-PnPSearchCrawlLog",
"Id": 727,
"Rank": 1
},
{
"CommandName": "Get-PnPSearchCrawlLog",
"Command": "Get-PnPSearchCrawlLog -Filter \"https://contoso-my.sharepoint.com/personal\"",
"Id": 728,
"Rank": 2
},
{
"CommandName": "Get-PnPSearchCrawlLog",
"Command": "Get-PnPSearchCrawlLog -ContentSource UserProfiles",
"Id": 729,
"Rank": 3
},
{
"CommandName": "Get-PnPSearchCrawlLog",
"Command": "Get-PnPSearchCrawlLog -ContentSource UserProfiles -Filter \"mikael\"",
"Id": 730,
"Rank": 4
},
{
"CommandName": "Get-PnPSearchCrawlLog",
"Command": "Get-PnPSearchCrawlLog -ContentSource Sites -LogLevel Error -RowLimit 10",
"Id": 731,
"Rank": 5
},
{
"CommandName": "Get-PnPSearchCrawlLog",
"Command": "Get-PnPSearchCrawlLog -EndDate (Get-Date).AddDays(-100)",
"Id": 732,
"Rank": 6
},
{
"CommandName": "Get-PnPSearchCrawlLog",
"Command": "Get-PnPSearchCrawlLog -RowLimit 3 -RawFormat",
"Id": 733,
"Rank": 7
},
{
"CommandName": "Get-PnPSearchExternalConnection",
"Command": "Get-PnPSearchExternalConnection",
Expand Down
148 changes: 2 additions & 146 deletions src/Commands/Search/GetSearchCrawlLog.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public class CrawlEntry

[Cmdlet(VerbsCommon.Get, "PnPSearchCrawlLog", DefaultParameterSetName = "Xml")]
[ApiNotAvailableUnderApplicationPermissions]
[Obsolete("The underlying API for this cmdlet has been deprecated by Microsoft and no longer returns results. This cmdlet will be removed in a future version.")]
public class GetSearchCrawlLog : PnPWebCmdlet
{
[Parameter(Mandatory = false)]
Expand Down Expand Up @@ -62,153 +63,8 @@ public class GetSearchCrawlLog : PnPWebCmdlet

protected override void ExecuteCmdlet()
{
try
{
var crawlLog = new DocumentCrawlLog(ClientContext, ClientContext.Site);
ClientContext.Load(crawlLog);

int contentSourceId;
switch (ContentSource)
{
case ContentSource.Sites:
contentSourceId = GetContentSourceIdForSites(crawlLog);
break;
case ContentSource.UserProfiles:
contentSourceId = GetContentSourceIdForUserProfiles(crawlLog);
break;
default:
throw new ArgumentOutOfRangeException();
}

string postFilter = string.Empty;
if (string.IsNullOrWhiteSpace(Filter) && ContentSource == ContentSource.Sites)
{
Filter = $"https://{GetHostName()}.sharepoint.{PnP.Framework.AuthenticationManager.GetSharePointDomainSuffix(Connection.AzureEnvironment)}";
}

int origLimit = RowLimit;
if (ContentSource == ContentSource.UserProfiles)
{
postFilter = Filter;
Filter = $"https://{GetHostName()}-my.sharepoint.{PnP.Framework.AuthenticationManager.GetSharePointDomainSuffix(Connection.AzureEnvironment)}";
RowLimit = MaxRows;
}

var logEntries = crawlLog.GetCrawledUrls(false, RowLimit, Filter, true, contentSourceId, (int)LogLevel, -1, StartDate, EndDate);
ClientContext.ExecuteQueryRetry();

if (RawFormat)
{
var entries = new List<object>();
foreach (var dictionary in logEntries.Value.Rows)
{
string url = System.Net.WebUtility.UrlDecode(dictionary["FullUrl"].ToString());
if (ContentSource == ContentSource.UserProfiles && contentSourceId == -1)
{
if (!url.Contains(":443/person")) continue;
}
if (string.IsNullOrWhiteSpace(postFilter) || url.Contains(postFilter))
{
entries.Add(ConvertToPSObject(dictionary));
}
}
WriteObject(entries.Take(origLimit), true);
}
else
{
var entries = new List<CrawlEntry>(logEntries.Value.Rows.Count);
foreach (var dictionary in logEntries.Value.Rows)
{
var entry = MapCrawlLogEntry(dictionary);
if (string.IsNullOrWhiteSpace(postFilter) || entry.Url.Contains(postFilter))
{
entries.Add(entry);
}
}

if (ContentSource == ContentSource.UserProfiles && contentSourceId == -1)
{
// Crawling has changed and uses one content source
// Need to apply post-filter to pull out profile entries only
entries =
entries.Where(e => System.Net.WebUtility.UrlDecode(e.Url.ToString()).ToLower().Contains(":443/person"))
.ToList();
}
WriteObject(entries.Take(origLimit).OrderByDescending(i => i.CrawlTime).ToList(), true);
}
}
catch (Exception e)
{
LogError($"Error: {e.Message}. Make sure you are granted access to the crawl log via the SharePoint search admin center at https://<tenant>-admin.sharepoint.com/_layouts/15/searchadmin/crawllogreadpermission.aspx");
}
throw new NotSupportedException("The underlying API for Get-PnPSearchCrawlLog has been deprecated by Microsoft and no longer returns results. This cmdlet will be removed in a future version.");
}

#region Helper functions

private string GetHostName()
{
return new Uri(ClientContext.Url).Host.Replace("-admin", "").Replace("-public", "").Replace("-my", "").Replace($".sharepoint.{PnP.Framework.AuthenticationManager.GetSharePointDomainSuffix(Connection.AzureEnvironment)}", "");
}

private int GetContentSourceIdForSites(DocumentCrawlLog crawlLog)
{
var hostName = GetHostName();
var spContent = crawlLog.GetCrawledUrls(false, 10, $"https://{hostName}.sharepoint.{PnP.Framework.AuthenticationManager.GetSharePointDomainSuffix(Connection.AzureEnvironment)}/sites", true, -1, (int)LogLevel.All, -1, DateTime.Now.AddDays(-100), DateTime.Now.AddDays(1));
ClientContext.ExecuteQueryRetry();
if (spContent.Value.Rows.Count > 0) return (int)spContent.Value.Rows.First()["ContentSourceID"];
return -1;
}

private int GetContentSourceIdForUserProfiles(DocumentCrawlLog crawlLog)
{
var hostName = GetHostName();
var peopleContent = crawlLog.GetCrawledUrls(false, 100, $"sps3s://{hostName}-my.sharepoint.{PnP.Framework.AuthenticationManager.GetSharePointDomainSuffix(Connection.AzureEnvironment)}", true, -1, (int)LogLevel.All, -1, DateTime.Now.AddDays(-100), DateTime.Now.AddDays(1));
ClientContext.ExecuteQueryRetry();
if (peopleContent.Value.Rows.Count > 0) return (int)peopleContent.Value.Rows.First()["ContentSourceID"];
return -1;
}

private static CrawlEntry MapCrawlLogEntry(Dictionary<string, object> dictionary)
{
var entry = new CrawlEntry
{
ItemId = (int)dictionary["URLID"],
ContentSourceId = (int)dictionary["ContentSourceID"],
Url = dictionary["FullUrl"].ToString(),
CrawlTime = (DateTime)dictionary["TimeStampUtc"]
};
long.TryParse(dictionary["LastRepositoryModifiedTime"] + "", out long ticks);
if (ticks != 0)
{
var itemDate = DateTime.FromFileTimeUtc(ticks);
entry.ItemTime = itemDate;
}
entry.LogLevel =
(LogLevel)Enum.Parse(typeof(LogLevel), dictionary["ErrorLevel"].ToString());


entry.Status = dictionary["StatusMessage"] + "";
entry.Status += dictionary["ErrorDesc"] + "";
var errorCode = int.Parse(dictionary["ErrorCode"]+"");
if (!string.IsNullOrWhiteSpace(entry.Status) || errorCode != 0)
{
entry.LogLevel = LogLevel.Warning;
}
return entry;
}

private object ConvertToPSObject(IDictionary<string, object> r)
{
PSObject res = new PSObject();
if (r != null)
{
foreach (var kvp in r)
{
res.Properties.Add(new PSNoteProperty(kvp.Key, kvp.Value));
}
}
return res;
}
#endregion
}
}
Loading