Based on MR simplified the logic

app/helpers/application_helper.rb

@@ -41,7 +41,7 @@ module ApplicationHelper
       end
     if project.avatar_url
-      image_tag project.avatar_url(use_asset_path: project.public?), options
+      image_tag project.avatar_url, options
     else # generated icon
       project_identicon(project, options)
     end

app/helpers/groups_helper.rb

-require 'uri'
-
 module GroupsHelper
-  include Gitlab::CurrentSettings
   def can_change_group_visibility_level?(group)
     can?(current_user, :change_visibility_level, group)
@@ -22,12 +19,7 @@ module GroupsHelper
     end
     if group.avatar_url
-      if group.private?
-        options[:use_original_source] = true
-        group.avatar_url(use_asset_path: false)
-      else
-        group.avatar_url
-      end
+      group.avatar_url
     else # No Avatar Icon
       ActionController::Base.helpers.image_path('no_group_avatar.png')
     end
@@ -107,11 +99,7 @@ module GroupsHelper
     link_to(group_path(group), class: "group-path #{'breadcrumb-item-text' unless for_dropdown} js-breadcrumb-item-text #{'hidable' if hidable}") do
       output =
         if (group.try(:avatar_url) || show_avatar) && !Rails.env.test?
-          if group.private?
-            group_icon(group, class: "avatar-tile", width: 15, height: 15, use_original_source: true)
-          else
-            group_icon(group, class: "avatar-tile", width: 15, height: 15)
-          end
+          group_icon(group, class: "avatar-tile", width: 15, height: 15)
         else
           ""
         end

app/models/concerns/avatarable.rb

 module Avatarable
   extend ActiveSupport::Concern
-  def avatar_path(only_path: true, use_asset_path: true)
+  def avatar_path(only_path: true)
     return unless self[:avatar].present?
     # If only_path is true then use the relative path of avatar.
@@ -11,7 +11,7 @@ module Avatarable
     # If asset_host is set then it is expected that assets are handled by a standalone host.
     # That means we do not want to get GitLab's relative_url_root option anymore.
-    host = (asset_host.present? && use_asset_path) ? asset_host : gitlab_host
+    host = (asset_host.present? && (!respond_to?(:public?) || public?)) ? asset_host : gitlab_host
     [host, avatar.url].join
   end