Added recycling based on number of free i-nodes

README.md

@@ -10,28 +10,37 @@ By default the Docker Engine listens under `/var/run/docker.sock`
 ```
 docker run -d \
-	-e LOW_FREE_SPACE=10G \
-	-e EXPECTED_FREE_SPACE=20G \
-	-e DEFAULT_TTL=10m \
-	-e USE_DF=1 \
-	-v /var/run/docker.sock:/var/run/docker.sock \
-	--name=gitlab-runner-docker-cleanup \
-	quay.io/gitlab/gitlab-runner-docker-cleanup
+    -e LOW_FREE_SPACE=10G \
+    -e EXPECTED_FREE_SPACE=20G \
+    -e LOW_FREE_FILES_COUNT=1048576 \
+    -e EXPECTED_FREE_FILES_COUNT=2097152 \
+    -e DEFAULT_TTL=10m \
+    -e USE_DF=1 \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    --name=gitlab-runner-docker-cleanup \
+    quay.io/gitlab/gitlab-runner-docker-cleanup
 ```
+The above command will ensure to always have at least `10GB` of free disk space and at least `1M` of free files (i-nodes) on disk.
+
+The i-nodes is especially important when using Docker with `overlay` storage engine.
+More information about **i-node** problem [here](http://blog.cloud66.com/docker-with-overlayfs-first-impression/).
+
 ## Options
 You can configure GitLab Runner Docker Cleanup with environment variables:
 | Variable | Default | Description |
 | -------- | ------- | ----------- |
-| CHECK_PATH 			| / 	| The path which is used when checking disk usage |
-| LOW_FREE_SPACE 		| 1GB 	| When trigger the cache and image removal |
-| EXPECTED_FREE_SPACE 	| 2GB 	| How much the free space to cleanup |
-| USE_DF 				| false | Use a command line `df` tool to check disk space. Set to `false` when connecting to remote Docker Engine. Set to `true` when using with locally installed Docker Engine |
-| CHECK_INTERVAL 		| 10s 	| How often to check the disk space |
-| RETRY_INTERVAL 		| 30s 	| How long to wait before retrying in case of failure |
-| DEFAULT_TTL			| 1m 	| Minimum time to preserve a newly downloaded images or created caches |
+| CHECK_PATH                | /     | The path which is used when checking disk usage |
+| LOW_FREE_SPACE            | 1GB   | When trigger the cache and image removal |
+| EXPECTED_FREE_SPACE       | 2GB   | How much the free space to cleanup |
+| LOW_FREE_FILES_COUNT      | 131072| When the number of free files (i-nodes) runs below this value trigger the cache and image removal |
+| EXPECTED_FREE_FILES_COUNT | 262144| How many free files (i-nodes) to cleanup |
+| USE_DF                    | false | Use a command line `df` tool to check disk space. Set to `false` when connecting to remote Docker Engine. Set to `true` when using with locally installed Docker Engine |
+| CHECK_INTERVAL            | 10s   | How often to check the disk space |
+| RETRY_INTERVAL            | 30s   | How long to wait before retrying in case of failure |
+| DEFAULT_TTL               | 1m    | Minimum time to preserve a newly downloaded images or created caches |
 ## Automated build

cleanup.go

@@ -36,23 +36,32 @@ var internalImages = []string{
 var diskSpaceImage = "alpine"
 var opts = struct {
-	MonitorPath       string        `long:"check-path" description:"Path to monitor when verifying disk space" env:"CHECK_PATH"`
-	LowFreeSpace      string        `long:"low-free-space" description:"When to trigger cleanup cycle" env:"LOW_FREE_SPACE"`
-	ExpectedFreeSpace string        `long:"expected-free-space" description:"How much free space to cleanup" env:"EXPECTED_FREE_SPACE"`
-	UseDf             bool          `long:"use-df" description:"Use 'df' to check disk space instead of docker container" env:"USE_DF"`
-	CheckInterval     time.Duration `long:"check-interval" description:"How often to check disk space?" env:"CHECK_INTERVAL"`
-	RetryInterval     time.Duration `long:"retry-interval" description:"How long to wait before trying again?" env:"RETRY_INTERVAL"`
-	DefaultTTL        time.Duration `long:"ttl" description:"Default minimum TTL for caches and images" env:"DEFAULT_TTL"`
+	MonitorPath            string        `long:"check-path" description:"Path to monitor when verifying disk space" env:"CHECK_PATH"`
+	LowFreeSpace           string        `long:"low-free-space" description:"When to trigger cleanup cycle" env:"LOW_FREE_SPACE"`
+	ExpectedFreeSpace      string        `long:"expected-free-space" description:"How much free space to cleanup" env:"EXPECTED_FREE_SPACE"`
+	LowFreeFilesCount      uint64        `long:"low-files-count" description:"Trigger cleanup cycle if number of i-nodes runs below this value" env:"LOW_FREE_FILES_COUNT"`
+	ExpectedFreeFilesCount uint64        `long:"expected-files-count" description:"How much free i-nodes to recycle" env:"EXPECTED_FREE_FILES_COUNT"`
+	UseDf                  bool          `long:"use-df" description:"Use 'df' to check disk space instead of docker container" env:"USE_DF"`
+	CheckInterval          time.Duration `long:"check-interval" description:"How often to check disk space?" env:"CHECK_INTERVAL"`
+	RetryInterval          time.Duration `long:"retry-interval" description:"How long to wait before trying again?" env:"RETRY_INTERVAL"`
+	DefaultTTL             time.Duration `long:"ttl" description:"Default minimum TTL for caches and images" env:"DEFAULT_TTL"`
 }{
 	"/",
 	"1GB",
 	"2GB",
+	128 * 1024,
+	256 * 1024,
 	false,
 	10 * time.Second,
 	30 * time.Second,
 	1 * time.Minute,
 }
+type DiskSpace struct {
+	BytesFree, BytesTotal uint64
+	FilesFree, FilesTotal uint64
+}
+
 type DockerClient interface {
 	Ping() error
 	ListImages(opts docker.ListImagesOptions) ([]docker.APIImages, error)
@@ -60,7 +69,7 @@ type DockerClient interface {
 	RemoveImage(name string) error
 	RemoveContainer(opts docker.RemoveContainerOptions) error
 	InspectContainer(id string) (*docker.Container, error)
-	DiskSpace(path string) (uint64, uint64, error)
+	DiskSpace(path string) (DiskSpace, error)
 }
 type CustomDockerClient struct {
@@ -103,24 +112,29 @@ var dockerCredentials docker_helpers.DockerCredentials
 var imagesUsed map[string]ImageInfo = make(map[string]ImageInfo)
 var cachesUsed map[string]CacheInfo = make(map[string]CacheInfo)
-func (c *CustomDockerClient) diskSpaceLocally(path string) (uint64, uint64, error) {
+func (c *CustomDockerClient) diskSpaceLocally(path string) (ds DiskSpace, err error) {
 	var stat syscall.Statfs_t
-	err := syscall.Statfs(path, &stat)
-	if err != nil {
-		return 0, 0, err
+	err = syscall.Statfs(path, &stat)
+	if err == nil {
+		ds = DiskSpace{
+			BytesFree:  stat.Bavail * uint64(stat.Bsize),
+			BytesTotal: stat.Blocks * uint64(stat.Bsize),
+			FilesFree:  stat.Ffree,
+			FilesTotal: stat.Files,
+		}
 	}
-	return stat.Bavail * uint64(stat.Bsize), stat.Blocks * uint64(stat.Bsize), nil
+	return
 }
-func (c *CustomDockerClient) diskSpaceRemotely(path string) (uint64, uint64, error) {
-	_, err := c.InspectImage(diskSpaceImage)
+func (c *CustomDockerClient) diskSpaceRemotely(path string) (ds DiskSpace, err error) {
+	_, err = c.InspectImage(diskSpaceImage)
 	if err != nil {
 		logrus.Debugln("Pulling", diskSpaceImage, "...")
-		err := c.PullImage(docker.PullImageOptions{
+		err = c.PullImage(docker.PullImageOptions{
 			Repository: diskSpaceImage,
 		}, docker.AuthConfiguration{})
 		if err != nil {
-			return 0, 0, err
+			return
 		}
 	}
@@ -129,12 +143,12 @@ func (c *CustomDockerClient) diskSpaceRemotely(path string) (uint64, uint64, err
 		Config: &docker.Config{
 			Image:        diskSpaceImage,
 			Entrypoint:   []string{"/bin/stat"},
-			Cmd:          []string{"-f", "-c%a %b %s", path},
+			Cmd:          []string{"-f", "-c%a %b %s %d %c", path},
 			AttachStdout: true,
 		},
 	})
 	if err != nil {
-		return 0, 0, err
+		return
 	}
 	defer c.RemoveContainer(docker.RemoveContainerOptions{
@@ -144,12 +158,12 @@ func (c *CustomDockerClient) diskSpaceRemotely(path string) (uint64, uint64, err
 	err = c.StartContainer(container.ID, nil)
 	if err != nil {
-		return 0, 0, err
+		return
 	}
 	errorCode, err := c.WaitContainer(container.ID)
 	if err != nil || errorCode != 0 {
-		return 0, 0, err
+		return
 	}
 	var buffer bytes.Buffer
@@ -160,19 +174,25 @@ func (c *CustomDockerClient) diskSpaceRemotely(path string) (uint64, uint64, err
 		Tail:         "1",
 	})
 	if err != nil {
-		return 0, 0, err
+		return
 	}
-	var freeBlocks, totalBlocks, blockSize uint64
-	_, err = fmt.Fscanln(&buffer, &freeBlocks, &totalBlocks, &blockSize)
+	var freeBlocks, totalBlocks, blockSize, freeFiles, totalFiles uint64
+	_, err = fmt.Fscanln(&buffer, &freeBlocks, &totalBlocks, &blockSize, &freeFiles, &totalFiles)
 	if err != nil {
-		return 0, 0, err
+		return
 	}
-	return uint64(freeBlocks * blockSize), uint64(totalBlocks * blockSize), nil
+	ds = DiskSpace{
+		BytesFree:  uint64(freeBlocks * blockSize),
+		BytesTotal: uint64(totalBlocks * blockSize),
+		FilesFree:  freeFiles,
+		FilesTotal: totalFiles,
+	}
+	return
 }
-func (c *CustomDockerClient) DiskSpace(path string) (uint64, uint64, error) {
+func (c *CustomDockerClient) DiskSpace(path string) (DiskSpace, error) {
 	if opts.UseDf {
 		return c.diskSpaceLocally(path)
 	} else {
@@ -340,7 +360,7 @@ func updateContainers(client DockerClient) error {
 	return nil
 }
-func doFreeSpace(client DockerClient, freeSpace uint64) error {
+func doFreeSpace(client DockerClient, freeSpace, freeFiles uint64) error {
 	images, err := client.ListImages(docker.ListImagesOptions{})
 	if err != nil {
 		logrus.Warningln("Failed to list images:", err)
@@ -357,11 +377,11 @@ func doFreeSpace(client DockerClient, freeSpace uint64) error {
 	var lastError error
 	for {
-		diskSpace, _, err := client.DiskSpace(opts.MonitorPath)
+		diskSpace, err := client.DiskSpace(opts.MonitorPath)
 		if err != nil {
 			return err
 		}
-		if diskSpace > freeSpace {
+		if diskSpace.BytesFree > freeSpace && diskSpace.FilesFree > freeFiles {
 			break
 		}
@@ -413,7 +433,7 @@ func doFreeSpace(client DockerClient, freeSpace uint64) error {
 	return lastError
 }
-func doCycle(client DockerClient, lowFreeSpace, freeSpace uint64) error {
+func doCycle(client DockerClient, lowFreeSpace, freeSpace, lowFreeFiles, freeFiles uint64) error {
 	err := updateImages(client)
 	if err != nil {
 		logrus.Warningln("Failed to update images:", err)
@@ -426,28 +446,42 @@ func doCycle(client DockerClient, lowFreeSpace, freeSpace uint64) error {
 		return err
 	}
-	diskSpace, _, err := client.DiskSpace(opts.MonitorPath)
+	diskSpace, err := client.DiskSpace(opts.MonitorPath)
 	if err != nil {
 		logrus.Warningln("Failed to verify disk space:", err)
 		return err
 	}
-	if diskSpace >= lowFreeSpace {
-		logrus.Debugln("Nothing to free. Current disk space", humanize.Bytes(diskSpace),
-			"is above the lower bound", humanize.Bytes(lowFreeSpace))
+	if diskSpace.BytesFree >= lowFreeSpace && diskSpace.FilesFree >= lowFreeFiles {
+		if diskSpace.BytesFree >= lowFreeSpace {
+			logrus.Debugln("Nothing to free. Current free disk space", humanize.Bytes(diskSpace.BytesFree),
+				"is above the lower bound", humanize.Bytes(lowFreeSpace))
+		}
+		if diskSpace.FilesFree >= lowFreeFiles {
+			logrus.Debugln("Nothing to free. Current free files count", diskSpace.FilesFree,
+				"is above the lower bound", lowFreeFiles)
+		}
 		return nil
 	}
-	logrus.Warningln("Freeing disk space. The disk space is below:", humanize.Bytes(diskSpace),
-		"trying to free to:", humanize.Bytes(freeSpace))
-	freeSpaceErr := doFreeSpace(client, freeSpace)
+	if diskSpace.BytesFree < lowFreeSpace {
+		logrus.Warningln("Freeing disk space. The disk space is below the lower bound:", humanize.Bytes(diskSpace.BytesFree),
+			"trying to free up to:", humanize.Bytes(freeSpace))
+	}
+	if diskSpace.FilesFree < lowFreeSpace {
+		logrus.Warningln("Freeing files count. The free file count is below the lower bound:", diskSpace.FilesFree,
+			"trying to free up to:", freeFiles)
+	}
+	freeSpaceErr := doFreeSpace(client, freeSpace, freeFiles)
 	if freeSpaceErr != nil {
 		logrus.Warningln("Failed to free disk space:", freeSpaceErr)
 	}
-	currentDiskSpace, _, err := client.DiskSpace(opts.MonitorPath)
+	currentDiskSpace, err := client.DiskSpace(opts.MonitorPath)
 	if err == nil {
-		logrus.Infoln("Freed:", humanize.Bytes(currentDiskSpace-diskSpace))
+		logrus.Infoln("Freed",
+			"bytes:", humanize.Bytes(currentDiskSpace.BytesFree-diskSpace.BytesFree),
+			"files:", currentDiskSpace.FilesFree-diskSpace.FilesFree)
 	}
 	return freeSpaceErr
@@ -483,7 +517,7 @@ func runCleanupTool(c *cli.Context) {
 			}
 		}
-		err = doCycle(dockerClient, lowFreeSpace, expectedFreeSpace)
+		err = doCycle(dockerClient, lowFreeSpace, expectedFreeSpace, opts.LowFreeFilesCount, opts.ExpectedFreeFilesCount)
 		if err == nil {
 			time.Sleep(opts.CheckInterval)
 		} else {

cleanup_test.go

@@ -20,6 +20,8 @@ type MockDockerClient struct {
 	links             []string
 	freeSpace         uint64
 	totalSpace        uint64
+	freeFiles         uint64
+	totalFiles        uint64
 }
 func (c *MockDockerClient) Ping() error {
@@ -32,7 +34,8 @@ func (c *MockDockerClient) RemoveImage(name string) error {
 	}
 	for _, image := range c.images {
 		if image.ID == name {
-			c.freeSpace -= uint64(image.Size)
+			c.freeSpace += uint64(image.Size)
+			c.freeFiles += uint64(image.Size / 4096)
 		}
 	}
 	c.removedImages = append(c.removedImages, name)
@@ -45,7 +48,8 @@ func (c *MockDockerClient) RemoveContainer(opts RemoveContainerOptions) error {
 	}
 	for _, container := range c.containers {
 		if container.ID == opts.ID {
-			c.freeSpace -= uint64(container.SizeRw)
+			c.freeSpace += uint64(container.SizeRw)
+			c.freeFiles += uint64(container.SizeRw / 4096)
 		}
 	}
 	c.removedContainers = append(c.removedContainers, opts.ID)
@@ -60,8 +64,13 @@ func (c *MockDockerClient) ListContainers(opts ListContainersOptions) ([]APICont
 	return c.containers, c.error
 }
-func (c *MockDockerClient) DiskSpace(path string) (uint64, uint64, error) {
-	return c.freeSpace, c.totalSpace, c.error
+func (c *MockDockerClient) DiskSpace(path string) (DiskSpace, error) {
+	return DiskSpace{
+		BytesFree:  c.freeSpace,
+		BytesTotal: c.totalSpace,
+		FilesFree:  c.freeFiles,
+		FilesTotal: c.totalFiles,
+	}, c.error
 }
 func (c *MockDockerClient) InspectContainer(id string) (*Container, error) {
@@ -357,13 +366,14 @@ func (s *CleanupSuite) TestMarksImage(c *C) {
 func (s *CleanupSuite) TestCycleWithEnoughDiskSpace(c *C) {
 	s.dockerClient.freeSpace = humanize.GByte
-	err := doCycle(s.dockerClient, humanize.MByte, humanize.GByte)
+	s.dockerClient.freeFiles = 100000
+	err := doCycle(s.dockerClient, humanize.MByte, humanize.GByte, 1000, 10000)
 	c.Assert(err, IsNil)
 }
 func (s *CleanupSuite) TestCycleUnableToCleanup(c *C) {
 	s.dockerClient.freeSpace = humanize.KByte
-	err := doCycle(s.dockerClient, humanize.MByte, humanize.GByte)
+	err := doCycle(s.dockerClient, humanize.MByte, humanize.GByte, 1000, 10000)
 	c.Assert(err, NotNil)
 	c.Assert(err.Error(), Matches, "no images or caches to delete")
 }
@@ -378,7 +388,7 @@ func (s *CleanupSuite) TestFreeingSpaceAndNotRemovingContainersWithInternalImage
 	err := updateImages(s.dockerClient)
 	c.Assert(err, IsNil)
-	err = doFreeSpace(s.dockerClient, 2*humanize.GByte)
+	err = doFreeSpace(s.dockerClient, 2*humanize.GByte, 100000)
 	c.Assert(err, NotNil)
 	c.Assert(s.dockerClient.removedImages, HasLen, 1)
 }
@@ -393,11 +403,26 @@ func (s *CleanupSuite) TestFreeingSpaceByRemovingImages(c *C) {
 	err := updateImages(s.dockerClient)
 	c.Assert(err, IsNil)
-	err = doFreeSpace(s.dockerClient, 2*humanize.GByte)
+	err = doFreeSpace(s.dockerClient, 2*humanize.GByte, 100000)
 	c.Assert(err, IsNil)
 	c.Assert(s.dockerClient.removedImages, HasLen, 2)
 }
+func (s *CleanupSuite) TestFreeingFilesByRemovingImages(c *C) {
+	s.dockerClient.images = []APIImages{
+		makeDockerImageWithSize("test", 40*humanize.MByte),
+	}
+	s.dockerClient.freeSpace = humanize.TByte
+	s.dockerClient.freeFiles = 500
+
+	err := updateImages(s.dockerClient)
+	c.Assert(err, IsNil)
+
+	err = doCycle(s.dockerClient, humanize.MByte, humanize.GByte, 1000, 10000)
+	c.Assert(err, IsNil)
+	c.Assert(s.dockerClient.removedImages, HasLen, 1)
+}
+
 func (s *CleanupSuite) TestFreeingSpaceByRemovingCaches(c *C) {
 	s.dockerClient.freeSpace = humanize.GByte
 	s.dockerClient.containers = []APIContainers{
@@ -408,7 +433,7 @@ func (s *CleanupSuite) TestFreeingSpaceByRemovingCaches(c *C) {
 	err := updateContainers(s.dockerClient)
 	c.Assert(err, IsNil)
-	err = doFreeSpace(s.dockerClient, 2*humanize.GByte)
+	err = doFreeSpace(s.dockerClient, 2*humanize.GByte, 100000)
 	c.Assert(err, IsNil)
 	c.Assert(s.dockerClient.removedContainers, HasLen, 2)
 }
@@ -423,7 +448,7 @@ func (s *CleanupSuite) TestFreeingSpaceAndIgnoringNonCacheContainers(c *C) {
 	err := updateContainers(s.dockerClient)
 	c.Assert(err, IsNil)
-	err = doFreeSpace(s.dockerClient, 2*humanize.GByte)
+	err = doFreeSpace(s.dockerClient, 2*humanize.GByte, 100000)
 	c.Assert(err, NotNil)
 	c.Assert(s.dockerClient.removedContainers, HasLen, 1)
 }