Fix bug with installing .NET templates

release_notes.md

@@ -1,12 +1,10 @@
-# Azure Functions CLI 4.2.1
+# Azure Functions CLI 4.2.2
 
 #### Host Version
 
 - Host Version: 4.1041.200
-- In-Proc Host Version: 4.41.100 (4.841.100,  4.641.100)
+- In-Proc Host Version: 4.41.100 (4.841.100, 4.641.100)
 
 #### Changes
 
-- Add support for .NET 10 isolated model (#4589)
-- Update log streaming to support both connection string and instrumentation Key (#4586)
-- Remove content of workers dir from minified versions (#4609)
+- Fix .NET template install bug (#4612)

src/Cli/func/Common/FileSystemHelpers.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 
 using System.IO.Abstractions;
@@ -127,6 +127,12 @@ public static string EnsureDirectory(string path)
             return path;
         }
 
+        public static bool EnsureDirectoryNotEmpty(string path)
+        {
+            return DirectoryExists(path) &&
+                Instance.Directory.EnumerateFileSystemEntries(path).Any();
+        }
+
         public static void DeleteDirectorySafe(string path, bool ignoreErrors = true)
         {
             DeleteFileSystemInfo(Instance.DirectoryInfo.FromDirectoryName(path), ignoreErrors);

src/Cli/func/Directory.Version.props

@@ -1,7 +1,7 @@
 <Project>
 
   <PropertyGroup>
-    <VersionPrefix>4.2.1</VersionPrefix>
+    <VersionPrefix>4.2.2</VersionPrefix>
     <VersionSuffix></VersionSuffix>
     <UpdateBuildNumber>true</UpdateBuildNumber>
   </PropertyGroup>

src/Cli/func/Helpers/DotnetHelpers.E2E.cs

@@ -0,0 +1,56 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+using Azure.Functions.Cli.Common;
+
+namespace Azure.Functions.Cli.Helpers
+{
+    // Partial class to hold E2E test related helpers
+    public static partial class DotnetHelpers
+    {
+        // Environment variable names to control custom hive usage in E2E tests
+        internal const string CustomHiveFlag = "FUNC_E2E_USE_CUSTOM_HIVE";
+        internal const string CustomHiveRoot = "FUNC_E2E_HIVE_ROOT";
+        internal const string CustomHiveKey = "FUNC_E2E_HIVE_KEY";
+
+        private static bool UseCustomTemplateHive() => string.Equals(Environment.GetEnvironmentVariable(CustomHiveFlag), "1", StringComparison.Ordinal);
+
+        private static string GetHiveRoot()
+        {
+            string root = Environment.GetEnvironmentVariable(CustomHiveRoot);
+
+            if (!string.IsNullOrWhiteSpace(root))
+            {
+                return root;
+            }
+
+            string coreToolsLocalDataPath = Utilities.EnsureCoreToolsLocalData();
+            return Path.Combine(coreToolsLocalDataPath, "dotnet-templates-custom-hives");
+        }
+
+        // By default, each worker runtime shares a hive. This can be overridden by setting the FUNC_E2E_HIVE_KEY
+        // environment variable to a custom value, which will cause a separate hive to be used.
+        private static string GetHivePath(WorkerRuntime workerRuntime)
+        {
+            string key = Environment.GetEnvironmentVariable(CustomHiveKey);
+            string leaf = !string.IsNullOrWhiteSpace(key) ? key : $"{workerRuntime.ToString().ToLowerInvariant()}-hive";
+            return Path.Combine(GetHiveRoot(), leaf);
+        }
+
+        private static bool TryGetCustomHiveArg(WorkerRuntime workerRuntime, out string customHiveArg)
+        {
+            customHiveArg = string.Empty;
+
+            if (!UseCustomTemplateHive())
+            {
+                return false;
+            }
+
+            string hive = GetHivePath(workerRuntime);
+            FileSystemHelpers.EnsureDirectory(hive);
+
+            customHiveArg = $" --debug:custom-hive \"{hive}\"";
+            return true;
+        }
+    }
+}

src/Cli/func/Helpers/DotnetHelpers.cs

@@ -1,22 +1,31 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 
-using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Text;
+using System.Text.RegularExpressions;
 using Azure.Functions.Cli.Common;
 using Colors.Net;
 using Microsoft.Azure.WebJobs.Extensions.Http;
 using static Azure.Functions.Cli.Common.OutputTheme;
 
 namespace Azure.Functions.Cli.Helpers
 {
-    public static class DotnetHelpers
+    public static partial class DotnetHelpers
     {
-        private const string WebJobsTemplateBasePackId = "Microsoft.Azure.WebJobs";
+        private const string InProcTemplateBasePackId = "Microsoft.Azure.WebJobs";
         private const string IsolatedTemplateBasePackId = "Microsoft.Azure.Functions.Worker";
-        private const string TemplatesLockFileName = "func_dotnet_templates.lock";
-        private static readonly Lazy<Task<HashSet<string>>> _installedTemplatesList = new(GetInstalledTemplatePackageIds);
+
+        /// <summary>
+        /// Gets or sets test hook to intercept 'dotnet new' invocations for unit tests.
+        /// If null, real process execution is used.
+        /// </summary>
+        internal static Func<string, Task<int>> RunDotnetNewFunc { get; set; } = null;
+
+        private static Task<int> RunDotnetNewAsync(string args)
+            => (RunDotnetNewFunc is not null)
+                ? RunDotnetNewFunc(args)
+                : new Executable("dotnet", args).RunAsync();
 
         public static void EnsureDotnet()
         {
@@ -64,7 +73,18 @@ public static async Task<string> DetermineTargetFramework(string projectDirector
                 throw new CliException($"Can not determine target framework for dotnet project at ${projectDirectory}");
             }
 
-            return output.ToString();
+            // Extract the target framework moniker (TFM) from the output using regex pattern matching
+            var outputString = output.ToString();
+
+            // Look for a line that looks like a target framework moniker
+            var tfm = TargetFrameworkHelper.TfmRegex.Match(outputString);
+
+            if (!tfm.Success)
+            {
+                throw new CliException($"Could not parse target framework from output: {outputString}");
+            }
+
+            return tfm.Value;
         }
 
         public static async Task DeployDotnetProject(string name, bool force, WorkerRuntime workerRuntime, string targetFramework = "")
@@ -78,7 +98,8 @@ await TemplateOperationAsync(
                     var connectionString = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
                         ? $"--StorageConnectionStringValue \"{Constants.StorageEmulatorConnectionString}\""
                         : string.Empty;
-                    var exe = new Executable("dotnet", $"new func {frameworkString} --AzureFunctionsVersion v4 --name {name} {connectionString} {(force ? "--force" : string.Empty)}");
+                    TryGetCustomHiveArg(workerRuntime, out string customHive);
+                    var exe = new Executable("dotnet", $"new func {frameworkString} --AzureFunctionsVersion v4 --name {name} {connectionString} {(force ? "--force" : string.Empty)}{customHive}");
                     var exitCode = await exe.RunAsync(o => { }, e => ColoredConsole.Error.WriteLine(ErrorColor(e)));
                     if (exitCode != 0)
                     {
@@ -109,7 +130,8 @@ await TemplateOperationAsync(
                         }
                     }
 
-                    var exe = new Executable("dotnet", exeCommandArguments);
+                    TryGetCustomHiveArg(workerRuntime, out string customHive);
+                    var exe = new Executable("dotnet", exeCommandArguments + customHive);
                     string dotnetNewErrorMessage = string.Empty;
                     var exitCode = await exe.RunAsync(o => { }, e =>
                     {
@@ -277,119 +299,110 @@ public static string GetCsprojOrFsproj()
             }
         }
 
-        private static async Task TemplateOperationAsync(Func<Task> action, WorkerRuntime workerRuntime)
+        internal static async Task TemplateOperationAsync(Func<Task> action, WorkerRuntime workerRuntime)
         {
             EnsureDotnet();
 
+            // If we have enabled custom hives (for E2E tests), install templates there and run the action
+            if (UseCustomTemplateHive())
+            {
+                await EnsureTemplatesInCustomHiveAsync(action, workerRuntime);
+                return;
+            }
+
+            // Default CLI behaviour: Templates are installed globally, so we need to install/uninstall them around the action
             if (workerRuntime == WorkerRuntime.DotnetIsolated)
             {
-                await EnsureIsolatedTemplatesInstalled();
+                await EnsureIsolatedTemplatesInstalledAsync(action);
             }
             else
             {
-                await EnsureWebJobsTemplatesInstalled();
+                await EnsureInProcTemplatesInstalledAsync(action);
             }
-
-            await action();
         }
 
-        private static async Task EnsureIsolatedTemplatesInstalled()
+        private static async Task EnsureTemplatesInCustomHiveAsync(Func<Task> action, WorkerRuntime workerRuntime)
         {
-            if (AreDotnetTemplatePackagesInstalled(await _installedTemplatesList.Value, WebJobsTemplateBasePackId))
-            {
-                await UninstallWebJobsTemplates();
-            }
-
-            if (AreDotnetTemplatePackagesInstalled(await _installedTemplatesList.Value, IsolatedTemplateBasePackId))
+            // If the custom hive already has templates installed, just run the action and skip installation
+            string hivePackagesDir = Path.Combine(GetHivePath(workerRuntime), "packages");
+            if (FileSystemHelpers.EnsureDirectoryNotEmpty(hivePackagesDir))
             {
+                await action();
                 return;
             }
 
-            await FileLockHelper.WithFileLockAsync(TemplatesLockFileName, InstallIsolatedTemplates);
+            // Install only, no need to uninstall as we are using a custom hive
+            Func<Task> installTemplates = workerRuntime == WorkerRuntime.DotnetIsolated
+                ? InstallIsolatedTemplates
+                : InstallInProcTemplates;
+
+            await installTemplates();
+            await action();
         }
 
-        private static async Task EnsureWebJobsTemplatesInstalled()
+        private static async Task EnsureIsolatedTemplatesInstalledAsync(Func<Task> action)
         {
-            if (AreDotnetTemplatePackagesInstalled(await _installedTemplatesList.Value, IsolatedTemplateBasePackId))
+            try
             {
-                await UninstallIsolatedTemplates();
-            }
+                // Uninstall any existing webjobs templates, as they conflict with isolated templates
+                await UninstallInProcTemplates();
 
-            if (AreDotnetTemplatePackagesInstalled(await _installedTemplatesList.Value, WebJobsTemplateBasePackId))
+                // Install the latest isolated templates
+                await InstallIsolatedTemplates();
+                await action();
+            }
+            finally
             {
-                return;
+                await UninstallIsolatedTemplates();
             }
-
-            await FileLockHelper.WithFileLockAsync(TemplatesLockFileName, InstallWebJobsTemplates);
-        }
-
-        internal static bool AreDotnetTemplatePackagesInstalled(HashSet<string> templates, string packageIdPrefix)
-        {
-            var hasProjectTemplates = templates.Contains($"{packageIdPrefix}.ProjectTemplates", StringComparer.OrdinalIgnoreCase);
-            var hasItemTemplates = templates.Contains($"{packageIdPrefix}.ItemTemplates", StringComparer.OrdinalIgnoreCase);
-
-            return hasProjectTemplates && hasItemTemplates;
         }
 
-        private static async Task<HashSet<string>> GetInstalledTemplatePackageIds()
+        private static async Task EnsureInProcTemplatesInstalledAsync(Func<Task> action)
         {
-            var exe = new Executable("dotnet", "new uninstall", shareConsole: false);
-            var output = new StringBuilder();
-            var exitCode = await exe.RunAsync(o => output.AppendLine(o), e => output.AppendLine(e));
+            try
+            {
+                // Uninstall any existing isolated templates, as they conflict with webjobs templates
+                await UninstallIsolatedTemplates();
 
-            if (exitCode != 0)
+                // Install the latest webjobs templates
+                await InstallInProcTemplates();
+                await action();
+            }
+            finally
             {
-                throw new CliException("Failed to get list of installed template packages");
+                await UninstallInProcTemplates();
             }
+        }
 
-            var lines = output.ToString()
-                .Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries);
-
-            var packageIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-
-            const string uninstallPrefix = "dotnet new uninstall ";
+        private static string[] GetNupkgFiles(string templatesPath)
+        {
+            var templatesLocation = Path.Combine(
+                   Path.GetDirectoryName(AppContext.BaseDirectory),
+                   Path.Combine(templatesPath));
 
-            foreach (var line in lines)
+            if (!FileSystemHelpers.DirectoryExists(templatesLocation))
             {
-                var trimmed = line.Trim();
-
-                if (trimmed.StartsWith(uninstallPrefix, StringComparison.OrdinalIgnoreCase))
-                {
-                    var packageId = trimmed.Substring(uninstallPrefix.Length).Trim();
-                    if (!string.IsNullOrWhiteSpace(packageId))
-                    {
-                        packageIds.Add(packageId);
-                    }
-                }
+                throw new CliException($"Can't find templates location. Looked under '{templatesLocation}'");
             }
 
-            return packageIds;
+            return Directory.GetFiles(templatesLocation, "*.nupkg", SearchOption.TopDirectoryOnly);
         }
 
-        private static Task UninstallIsolatedTemplates() => DotnetTemplatesAction("uninstall", nugetPackageList: [$"{IsolatedTemplateBasePackId}.ProjectTemplates", $"{IsolatedTemplateBasePackId}.ItemTemplates"]);
+        private static Task InstallIsolatedTemplates() => DotnetTemplatesAction("install", WorkerRuntime.DotnetIsolated, Path.Combine("templates", $"net-isolated"));
 
-        private static Task UninstallWebJobsTemplates() => DotnetTemplatesAction("uninstall", nugetPackageList: [$"{WebJobsTemplateBasePackId}.ProjectTemplates", $"{WebJobsTemplateBasePackId}.ItemTemplates"]);
+        private static Task UninstallIsolatedTemplates() => DotnetTemplatesAction("uninstall", WorkerRuntime.DotnetIsolated, nugetPackageList: [$"{IsolatedTemplateBasePackId}.ProjectTemplates", $"{IsolatedTemplateBasePackId}.ItemTemplates"]);
 
-        private static Task InstallWebJobsTemplates() => DotnetTemplatesAction("install", "templates");
+        private static Task InstallInProcTemplates() => DotnetTemplatesAction("install", WorkerRuntime.Dotnet, "templates");
 
-        private static Task InstallIsolatedTemplates() => DotnetTemplatesAction("install", Path.Combine("templates", $"net-isolated"));
+        private static Task UninstallInProcTemplates() => DotnetTemplatesAction("uninstall", WorkerRuntime.Dotnet, nugetPackageList: [$"{InProcTemplateBasePackId}.ProjectTemplates", $"{InProcTemplateBasePackId}.ItemTemplates"]);
 
-        private static async Task DotnetTemplatesAction(string action, string templateDirectory = null, string[] nugetPackageList = null)
+        private static async Task DotnetTemplatesAction(string action, WorkerRuntime workerRuntime, string templateDirectory = null, string[] nugetPackageList = null)
         {
             string[] list;
 
             if (!string.IsNullOrEmpty(templateDirectory))
             {
-                var templatesLocation = Path.Combine(
-                    Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location),
-                    templateDirectory);
-
-                if (!FileSystemHelpers.DirectoryExists(templatesLocation))
-                {
-                    throw new CliException($"Can't find templates location. Looked under '{templatesLocation}'");
-                }
-
-                list = Directory.GetFiles(templatesLocation, "*.nupkg", SearchOption.TopDirectoryOnly);
+                list = GetNupkgFiles(templateDirectory);
             }
             else
             {
@@ -398,8 +411,9 @@ private static async Task DotnetTemplatesAction(string action, string templateDi
 
             foreach (var nupkg in list)
             {
-                var exe = new Executable("dotnet", $"new {action} \"{nupkg}\"");
-                await exe.RunAsync();
+                TryGetCustomHiveArg(workerRuntime, out string customHive);
+                var args = $"new {action} \"{nupkg}\" {customHive}";
+                await RunDotnetNewAsync(args);
             }
         }
     }