diff --git a/.circleci/config.yml b/.circleci/config.yml index 3232389..82c342f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,8 +23,7 @@ jobs: - run: curl -L -s https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 -o /go/bin/dep && chmod +x /go/bin/dep - run: dep ensure -vendor-only - run: curl -L -s https://github.com/alecthomas/gometalinter/releases/download/v2.0.5/gometalinter-2.0.5-linux-amd64.tar.gz | tar xvfz - -C /go/bin/ --strip 1 -# - run: gometalinter --disable-all --enable=errcheck --enable=vet --enable=vetshadow --vendor ./... - - run: gometalinter --disable-all --enable=vet --enable=vetshadow --vendor ./... + - run: gometalinter --disable-all --enable=megacheck --enable=golint --enable=unconvert --enable=vet --enable=vetshadow --vendor ./... deploy: <<: *defaults diff --git a/Gopkg.lock b/Gopkg.lock index 44c29a3..e110f7f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -11,18 +11,19 @@ branch = "master" name = "github.com/davecgh/go-spew" packages = ["spew"] - revision = "87df7c60d5820d0f8ae11afede5aa52325c09717" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" [[projects]] name = "github.com/fsnotify/fsnotify" packages = ["."] - revision = "4da3e2cfbabc9f751898f250b49f2439785783a1" + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" [[projects]] - branch = "master" name = "github.com/golang/protobuf" packages = ["proto"] - revision = "bbd03ef6da3a115852eaf24c8a1c46aeb39aa175" + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" [[projects]] branch = "master" @@ -35,6 +36,7 @@ revision = "36ee7e946282a3fb1cfecd476ddc9b35d8847e42" [[projects]] + branch = "master" name = "github.com/hashicorp/hcl" packages = [ ".", @@ -47,7 +49,7 @@ "json/scanner", "json/token" ] - revision = "392dba7d905ed5d04a5794ba89f558b27e2ba1ca" + revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" [[projects]] name = "github.com/inconshreveable/mousetrap" @@ -58,19 +60,20 @@ [[projects]] name = "github.com/magiconair/properties" packages = ["."] - revision = "51463bfca2576e06c62a8504b5c0f06d61312647" + revision = "c3beff4c2358b44d0493c7dda585e7db7ff28ae6" + version = "v1.7.6" [[projects]] - branch = "master" name = "github.com/mattn/go-runewidth" packages = ["."] - revision = "97311d9f7767e3d6f422ea06661bc2c7a19e8a5d" + revision = "9e777a8366cce605130a531d2cd6363d07ad7317" + version = "v0.0.2" [[projects]] - branch = "master" name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] - revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" + version = "v1.0.0" [[projects]] branch = "master" @@ -79,20 +82,22 @@ revision = "b8bc1bf767474819792c23f32d8286a45736f1c6" [[projects]] + branch = "master" name = "github.com/mitchellh/mapstructure" packages = ["."] - revision = "d0303fe809921458f417bcf828397a65db30a7e4" - -[[projects]] - name = "github.com/pelletier/go-buffruneio" - packages = ["."] - revision = "c37440a7cf42ac63b919c752ca73a85067e05992" - version = "v0.2.0" + revision = "00c29f56e2386353d58c599509e8dc3801b0d716" [[projects]] name = "github.com/pelletier/go-toml" packages = ["."] - revision = "fe7536c3dee2596cdd23ee9976a17c22bdaae286" + revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" + version = "v1.1.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" [[projects]] name = "github.com/prometheus/client_golang" @@ -100,7 +105,8 @@ "prometheus", "prometheus/promhttp" ] - revision = "a40133b69fbd73ee655606a9bf5f8b9b9bf758dd" + revision = "c5b7fccd204277076155f10851dad72b76a49317" + version = "v0.8.0" [[projects]] branch = "master" @@ -127,7 +133,7 @@ "nfs", "xfs" ] - revision = "282c8707aa210456a825798969cc27edda34992a" + revision = "75f2d6163c7a100bed6e971044ea3de30ee3a678" [[projects]] branch = "master" @@ -147,23 +153,26 @@ ".", "mem" ] - revision = "9be650865eab0c12963d8753212f4f9c66cdcf12" + revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" + version = "v1.0.2" [[projects]] name = "github.com/spf13/cast" packages = ["."] - revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4" - version = "v1.1.0" + revision = "8965335b8c7107321228e3e3702cab9832751bac" + version = "v1.2.0" [[projects]] name = "github.com/spf13/cobra" packages = ["."] - revision = "8f5946caaeeff40a98d67f60c25e89c3525038a3" + revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" + version = "v0.0.1" [[projects]] + branch = "master" name = "github.com/spf13/jwalterweatherman" packages = ["."] - revision = "0efa5202c04663c757d84f90f5219c1250baf94f" + revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" [[projects]] name = "github.com/spf13/pflag" @@ -174,7 +183,14 @@ [[projects]] name = "github.com/spf13/viper" packages = ["."] - revision = "a1ecfa6a20bd4ef9e9caded262ee1b1b26847675" + revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" + version = "v1.0.0" + +[[projects]] + name = "github.com/stretchr/testify" + packages = ["assert"] + revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0" + version = "v1.1.4" [[projects]] branch = "master" @@ -186,7 +202,7 @@ branch = "master" name = "golang.org/x/crypto" packages = ["ssh/terminal"] - revision = "650f4a345ab4e5b245a3034b110ebc7299e68186" + revision = "8c653846df49742c4c85ec37e5d9f8d3ba657895" [[projects]] branch = "master" @@ -195,7 +211,7 @@ "unix", "windows" ] - revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" + revision = "f6cff0780e542efa0c8e864dc8fa522808f6a598" [[projects]] name = "golang.org/x/text" @@ -207,16 +223,18 @@ "unicode/cldr", "unicode/norm" ] - revision = "9e2f80a6ba7ed4ba13e0cd4b1f094bf916875735" + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] - revision = "a83829b6f1293c91addabc89d0571c246397bbf4" + revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" + version = "v2.1.1" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "679d8f3e08c6fb8c5ef8eb95cedf7aec17654ae00051c78638c88c3731b1dc43" + inputs-digest = "aad3e0135314cde464ff9dc972643cf86f1f88e1dee72e091c7c4fb46a686e13" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 3a19740..eb537f2 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -52,3 +52,7 @@ [[constraint]] name = "github.com/speps/go-hashids" version = "1.0.0" + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.1.4" diff --git a/README.md b/README.md index 7bf3b1e..6d719de 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Inline docs](http://inch-ci.org/github/hipages/php-fpm_exporter.svg?branch=master)](http://inch-ci.org/github/hipages/php-fpm_exporter) [![Maintainability](https://api.codeclimate.com/v1/badges/52f9e1f0388e8aa38bfe/maintainability)](https://codeclimate.com/github/hipages/php-fpm_exporter/maintainability) [![Test Coverage](https://api.codeclimate.com/v1/badges/52f9e1f0388e8aa38bfe/test_coverage)](https://codeclimate.com/github/hipages/php-fpm_exporter/test_coverage) +[![Docker Pulls](https://img.shields.io/docker/pulls/hipages/php-fpm_exporter.svg)](https://hub.docker.com/r/hipages/php-fpm_exporter/) A [prometheus](https://prometheus.io/) exporter for PHP-FPM. The exporter connects directly to PHP-FPM and exports the metrics via HTTP. @@ -38,6 +39,23 @@ The `server` command runs the server required for prometheus to retrieve the sta | `--phpfpm.fix-process-count` | Enable to calculate process numbers via php-fpm_exporter since PHP-FPM sporadically reports wrong active/idle/total process numbers. | `PHP_FPM_FIX_PROCESS_COUNT`| `false` | | `--log.level` | Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal] (default "error") | `PHP_FPM_LOG_LEVEL` | info | +### Why `--phpfpm.fix-process-count`? + +`php-fpm_exporter` implements an option to "fix" the reported metrics based on the provided processes list by PHP-FPM. + +We have seen PHP-FPM provide metrics (e.g. active processes) which don't match reality. +Specially `active processes` being larger than `max_children` and the actual number of running processes on the host. +Looking briefly at the source code of PHP-FPM it appears a scoreboard is being kept and the values are increased/decreased once an action is executed. +The metric `active processes` is also an accumulation of multiple states (e.g. Reading headers, Getting request information, Running). +Which shouldn't matter and `active processes` should still be equal or lower to `max_children`. + +`--phpfpm.fix-process-count` will emulate PHP-FPMs implementation including the accumulation of multiple states. + +If you like to have a more granular reporting please use `phpfpm_process_state`. + +* https://bugs.php.net/bug.php?id=76003 +* https://stackoverflow.com/questions/48961556/can-active-processes-be-larger-than-max-children-for-php-fpm + ### CLI Examples * Retrieve information from PHP-FPM running on `127.0.0.1:9000` with status endpoint being `/status` diff --git a/cmd/root.go b/cmd/root.go index 9899ccc..0ec9ce8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,11 +27,10 @@ import ( var log = logrus.New() -var ( - cfgFile string - logLevel string - Version string -) +// Version that is being reported by the CLI +var Version string + +var cfgFile, logLevel string // RootCmd represents the base command when called without any subcommands var RootCmd = &cobra.Command{ diff --git a/phpfpm/exporter.go b/phpfpm/exporter.go index 08c0fb2..a817791 100644 --- a/phpfpm/exporter.go +++ b/phpfpm/exporter.go @@ -48,6 +48,7 @@ type Exporter struct { processRequests *prometheus.Desc processLastRequestMemory *prometheus.Desc processLastRequestCPU *prometheus.Desc + processState *prometheus.Desc } // NewExporter creates a new Exporter for a PoolManager and configures the necessary metrics. @@ -152,6 +153,12 @@ func NewExporter(pm PoolManager) *Exporter { "", []string{"pool", "pid"}, nil), + + processState: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "process_state"), + "The process state.", + []string{"pool", "pid", "state"}, + nil), } } @@ -171,12 +178,12 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { continue } - active, idle, total := CalculateProcessScoreboard(pool) - if active != pool.ActiveProcesses || idle != pool.IdleProcesses { + active, idle, total := CountProcessState(pool.Processes) + if !e.CalculateProcessScoreboard && (active != pool.ActiveProcesses || idle != pool.IdleProcesses) { log.Error("Inconsistent active and idle processes reported. Set `--fix-process-count` to have this calculated by php-fpm_exporter instead.") } - if e.CalculateProcessScoreboard == false { + if e.CalculateProcessScoreboard { active = pool.ActiveProcesses idle = pool.IdleProcesses total = pool.TotalProcesses @@ -197,13 +204,12 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { for _, process := range pool.Processes { pid := calculateProcessHash(process) + ch <- prometheus.MustNewConstMetric(e.processState, prometheus.GaugeValue, 1, pool.Name, pid, process.State) ch <- prometheus.MustNewConstMetric(e.processRequests, prometheus.CounterValue, float64(process.Requests), pool.Name, pid) ch <- prometheus.MustNewConstMetric(e.processLastRequestMemory, prometheus.GaugeValue, float64(process.LastRequestMemory), pool.Name, pid) - ch <- prometheus.MustNewConstMetric(e.processLastRequestCPU, prometheus.GaugeValue, float64(process.LastRequestCPU), pool.Name, pid) + ch <- prometheus.MustNewConstMetric(e.processLastRequestCPU, prometheus.GaugeValue, process.LastRequestCPU, pool.Name, pid) } } - - return } // Describe exposes the metric description to Prometheus diff --git a/phpfpm/phpfpm.go b/phpfpm/phpfpm.go index 06f3cba..43e98f0 100644 --- a/phpfpm/phpfpm.go +++ b/phpfpm/phpfpm.go @@ -29,8 +29,20 @@ import ( // PoolProcessRequestIdle defines a process that is idle. const PoolProcessRequestIdle string = "Idle" -// PoolProcessRequestIdle defines a process that is active. -const PoolProcessRequestActive string = "Running" +// PoolProcessRequestRunning defines a process that is running. +const PoolProcessRequestRunning string = "Running" + +// PoolProcessRequestFinishing defines a process that is about to finish. +const PoolProcessRequestFinishing string = "Finishing" + +// PoolProcessRequestReadingHeaders defines a process that is reading headers. +const PoolProcessRequestReadingHeaders string = "Reading headers" + +// PoolProcessRequestInfo defines a process that is getting request information. +const PoolProcessRequestInfo string = "Getting request informations" + +// PoolProcessRequestEnding defines a process that is about to end. +const PoolProcessRequestEnding string = "Ending" var log logger @@ -85,7 +97,17 @@ type PoolProcess struct { User string `json:"user"` Script string `json:"script"` LastRequestCPU float64 `json:"last request cpu"` - LastRequestMemory int `json:"last request memory"` + LastRequestMemory int64 `json:"last request memory"` +} + +// PoolProcessStateCounter holds the calculated metrics for pool processes. +type PoolProcessStateCounter struct { + Running int64 + Idle int64 + Finishing int64 + ReadingHeaders int64 + Info int64 + Ending int64 } // Add will add a pool to the pool manager based on the given URI. @@ -170,19 +192,21 @@ func (p *Pool) error(err error) error { return err } -func CalculateProcessScoreboard(p Pool) (active int64, idle int64, total int64) { - active = 0 - idle = 0 - total = 0 - - for idx := range p.Processes { - switch p.Processes[idx].State { - case PoolProcessRequestActive: +// CountProcessState return the calculated metrics based on the reported processes. +func CountProcessState(processes []PoolProcess) (active int64, idle int64, total int64) { + for idx := range processes { + switch processes[idx].State { + case PoolProcessRequestRunning: active++ case PoolProcessRequestIdle: idle++ + case PoolProcessRequestEnding: + case PoolProcessRequestFinishing: + case PoolProcessRequestInfo: + case PoolProcessRequestReadingHeaders: + active++ default: - log.Errorf("Unknown process state '%v'", p.Processes[idx].State) + log.Errorf("Unknown process state '%v'", processes[idx].State) } } diff --git a/phpfpm/phpfpm_test.go b/phpfpm/phpfpm_test.go new file mode 100644 index 0000000..c858396 --- /dev/null +++ b/phpfpm/phpfpm_test.go @@ -0,0 +1,95 @@ +// Copyright © 2018 Enrico Stahn +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package phpfpm + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCountProcessState(t *testing.T) { + processes := []PoolProcess{ + {State: PoolProcessRequestIdle}, + {State: PoolProcessRequestRunning}, + {State: PoolProcessRequestReadingHeaders}, + {State: PoolProcessRequestInfo}, + {State: PoolProcessRequestFinishing}, + {State: PoolProcessRequestEnding}, + } + + active, idle, total := CountProcessState(processes) + + assert.Equal(t, int64(2), active, "active processes") + assert.Equal(t, int64(1), idle, "idle processes") + assert.Equal(t, int64(3), total, "total processes") +} + +// https://github.com/hipages/php-fpm_exporter/issues/10 +func TestCannotUnmarshalNumberIssue10(t *testing.T) { + pool := Pool{} + content := []byte(`{ + "pool":"www", + "process manager":"dynamic", + "start time":1519474655, + "start since":302035, + "accepted conn":44144, + "listen queue":0, + "max listen queue":1, + "listen queue len":128, + "idle processes":1, + "active processes":1, + "total processes":2, + "max active processes":2, + "max children reached":0, + "slow requests":0, + "processes":[ + { + "pid":23, + "state":"Idle", + "start time":1519474655, + "start since":302035, + "requests":22071, + "request duration":295, + "request method":"GET", + "request uri":"/status?json&full", + "content length":0, + "user":"-", + "script":"-", + "last request cpu":0.00, + "last request memory":2097152 + }, + { + "pid":24, + "state":"Running", + "start time":1519474655, + "start since":302035, + "requests":22073, + "request duration":18446744073709550774, + "request method":"GET", + "request uri":"/status?json&full", + "content length":0, + "user":"-", + "script":"-", + "last request cpu":0.00, + "last request memory":0 + } + ] + }`) + + err := json.Unmarshal(content, &pool) + + assert.NotNil(t, err, err.Error()) +}