EPiServer Find Custom Indexing Job For Number of Sites

In one of our episerver projects we had 20 sites with one database. We are using epi find and epi find indexing job would never finish completely. During the indexing job after indexing couple of sites Epi Find indexing job would crash and stop the whole job.

It was becoming difficult to run the whole indexing job in one go. Therefore, I came up with other scheduling job to start from the site at which the job was failed instead of restarting from start.

This is the custom job to start epi find indexing where it left from.

    [ScheduledPlugIn(GUID = JobId, DisplayName = "Epi Find Incremental Indexing Job", DefaultEnabled = true)]
    public class EpiFindIncrementalIndexingJob : ScheduledJobBase
    {
        private const string LineBreak = "
";

        private const string JobId = "7107017B-F443-4E41-893D-62A9FAE69153";

        private readonly ILogger logger = LogManager.GetLogger(typeof(ReIndexNewsJob));

        private bool stopSignaled;

        private IScheduledJobLogRepository scheduledJobLogRepository;

        private ISiteDefinitionRepository siteDefinitionRepository;

        public EpiFindIncrementalIndexingJob()
        {
            IsStoppable = true;
            siteDefinitionRepository = ServiceLocator.Current.GetInstance();
            scheduledJobLogRepository = ServiceLocator.Current.GetInstance();
        }

        public override string Execute()
        {
            var executeDateTime = DateTime.UtcNow;
            using (var locker = DatabaseLock.Lock(Constants.Connections.EPiServerDB, nameof(EpiFindIncrementalIndexingJob), 0))
            {
                if (!locker.HasLock)
                    return "Error: failed to start. Job is already running.";

                // Call OnStatusChanged to periodically notify progress of job for manually started jobs
                OnStatusChanged($"Starting execution of {GetType()}");

                var message = new StringBuilder();
                var current = SiteDefinition.Current;
                var siteDefinitions = siteDefinitionRepository.List().Concat(new SiteDefinition[1] { SiteDefinition.Empty });

                try
                {
                    string getNameOfDefinition(SiteDefinition site) => site == SiteDefinition.Empty ? "Global assets and other data" : site.Name;

                    // Load the cached list of remaining sites to be indexed
                    var keyValue = ApplicationCache.Get(nameof(EpiFindIncrementalIndexingJob));
                    if (keyValue == null)
                    {
                        keyValue = new KeyValueItem()
                        {
                            Key = nameof(EpiFindIncrementalIndexingJob),
                            Updated = DateTime.Now,
                            Type = nameof(EpiFindIncrementalIndexingJob),
                            Value = string.Empty
                        };
                    }

                    var remaining = new HashSet();
                    foreach (var value in keyValue.Value.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
                        remaining.Add(value);

                    if (remaining.IsNullOrEmpty())
                        siteDefinitions.ToList().ForEach(s => remaining.Add(s.Id.ToString()));
                    else
                        siteDefinitions = siteDefinitions.Where(s => remaining.Contains(s.Id.ToString()));

                    stopSignaled = false;
                    foreach (SiteDefinition siteDefinition in siteDefinitions)
                    {
                        try
                        {
                            SiteDefinition.Current = siteDefinition;

                            var statusReport = new StringBuilder();

                            // ReIndex the current site (SiteDefinition.Current)
                            ContentIndexer.ReIndexResult reIndexResult = ContentIndexer.Instance.ReIndex(
                                status =>
                                {
                                    if (status.IsError)
                                    {
                                        string errorMessage = status.Message.StripHtml();
                                        if (errorMessage.Length > 0)
                                            statusReport.Append($"{errorMessage}");
                                    }
                                    OnStatusChanged($"Indexing job [{getNameOfDefinition(SiteDefinition.Current)}] [content]: {status.Message.StripHtml()}");
                                },
                                () => stopSignaled);

                             var siteMessage = $"Indexing job [{getNameOfDefinition(SiteDefinition.Current)}] [content]: {reIndexResult.PrintReport().StripHtml().Replace(Environment.NewLine, string.Empty)}{LineBreak}";

                            AddLogEntry(siteMessage);
                            message.Append(siteMessage);

                            if (stopSignaled)
                                return message.Append("Scheduled job was stopped").ToString();

                            if (statusReport.Length > 0)
                                message.Append($"{statusReport.ToString()}{LineBreak}");

                            // Update list of remaining sites to support resuming indexing, rather than restarting
                            remaining.RemoveWhere(s => siteDefinition.Id.ToString().Equals(s));
                            keyValue.Value = string.Join(",", remaining);
                            ApplicationCache.Set(nameof(EpiFindIncrementalIndexingJob), keyValue, Constants.Cache.OneYear);
                        }
                        catch (Exception ex)
                        {
                            message.Append($"{ex.ToString()}{LineBreak}");
                            LogException(ex);
                        }
                    }

                    // HACK: Use reflection to trigger the re-index of external best bets
                    var bestBetType = typeof(EPiServer.Find.Cms.BestBets.BestBetSelectorExtensions).Assembly.GetTypes()
                        .SingleOrDefault(x => x.FullName.EndsWith("BestBets.ExternalUrlBestBetHandlers", StringComparison.InvariantCultureIgnoreCase));

                    if (bestBetType != null)
                    {
                        var reIndexMethod = bestBetType.GetMethod("ReindexExternalUrlBestBets", BindingFlags.Static | BindingFlags.NonPublic);
                        if (reIndexMethod != null)
                        {
                            string result = reIndexMethod.Invoke(null, null)?.ToString();
                            message.Append($"{result}{LineBreak}");
                        }
                    }
                }
                catch (Exception ex)
                {
                    message.Append($"{ex.ToString()}{LineBreak}");
                    LogException(ex);
                }
                finally
                {
                    SiteDefinition.Current = current;
                }

                return message.ToString();
            }
        }

        public override void Stop()
        {
            stopSignaled = true;
        }

        private void AddLogEntry(string message)
        {
            scheduledJobLogRepository.LogAsync(new Guid(JobId), new ScheduledJobLogItem
            {
                CompletedUtc = DateTime.UtcNow,
                Message = message,
                Status = ScheduledJobExecutionStatus.Succeeded,
            }).GetAwaiter().GetResult();
        }

        private void LogException(Exception exception)
        {
            logger.Critical($"Exception occurred in {nameof(EpiFindIncrementalIndexingJob)}", exception);
        }
    }
}