##
# Sample ceph ceph.conf file.
##
# This file defines cluster membership, the various locations
# that Ceph stores data, and any other runtime options.

# If a 'host' is defined for a daemon, the init.d start/stop script will
# verify that it matches the hostname (or else ignore it).  If it is
# not defined, it is assumed that the daemon is intended to start on
# the current host (e.g., in a setup with a startup.conf on each
# node).

## Metavariables
# $cluster    ; Expands to the Ceph Storage Cluster name. Useful
#             ; when running multiple Ceph Storage Clusters
#             ; on the same hardware.
#             ; Example: /etc/ceph/$cluster.keyring
#             ; (Default: ceph)
#
# $type       ; Expands to one of mds, osd, or mon, depending on
#             ; the type of the instant daemon.
#             ; Example: /var/lib/ceph/$type
#
# $id         ; Expands to the daemon identifier. For osd.0, this
#             ; would be 0; for mds.a, it would be a.
#             ; Example: /var/lib/ceph/$type/$cluster-$id
#
# $host       ; Expands to the host name of the instant daemon.
#
# $name       ; Expands to $type.$id.
#             ; Example: /var/run/ceph/$cluster-$name.asok

[global]
### http://ceph.com/docs/master/rados/configuration/general-config-ref/

    fsid                       = b4b2e571-fbbf-4ff3-a9f8-ab80f08b7fe6    # use `uuidgen` to generate your own UUID
    public network             = 192.168.0.0/24
    cluster network            = 192.168.0.0/24

    # Each running Ceph daemon has a running process identifier (PID) file.
    # The PID file is generated upon start-up.
    # Type: String (optional)
    # (Default: N/A). The default path is /var/run/$cluster/$name.pid.
    pid file                   = /var/run/ceph/$name.pid

    # If set, when the Ceph Storage Cluster starts, Ceph sets the max open fds
    # at the OS level (i.e., the max # of file descriptors).
    # It helps prevents Ceph OSD Daemons from running out of file descriptors.
    # Type: 64-bit Integer (optional)
    # (Default: 0)
    max open files             = 131072


### http://ceph.com/docs/master/rados/operations/authentication
### http://ceph.com/docs/master/rados/configuration/auth-config-ref/

    # If enabled, the Ceph Storage Cluster daemons (i.e., ceph-mon, ceph-osd,
    # and ceph-mds) must authenticate with each other.
    # Type: String (optional); Valid settings are "cephx" or "none".
    # (Default: cephx)
    auth cluster required      = cephx

    # If enabled, the Ceph Storage Cluster daemons require Ceph Clients to
    # authenticate with the Ceph Storage Cluster in order to access Ceph
    # services.
    # Type: String (optional); Valid settings are "cephx" or "none".
    # (Default: cephx)
    auth service required      = cephx

    # If enabled, the Ceph Client requires the Ceph Storage Cluster to
    # authenticate with the Ceph Client.
    # Type: String (optional); Valid settings are "cephx" or "none".
    # (Default: cephx)
    auth client required       = cephx

    # If set to true, Ceph requires signatures on all message traffic between
    # the Ceph Client and the Ceph Storage Cluster, and between daemons
    # comprising the Ceph Storage Cluster.
    # Type: Boolean (optional)
    # (Default: false)
    cephx require signatures   = true

    # kernel RBD client do not support authentication yet:
    cephx cluster require signatures = true
    cephx service require signatures = false

    # The path to the keyring file.
    # Type: String (optional)
    # Default: /etc/ceph/$cluster.$name.keyring,/etc/ceph/$cluster.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin
    keyring                  = /etc/ceph/$cluster.$name.keyring


### http://ceph.com/docs/master/rados/configuration/pool-pg-config-ref/


    ## Replication level, number of data copies.
    # Type: 32-bit Integer
    # (Default: 3)
    osd pool default size      = 3

    ## Replication level in degraded state, less than 'osd pool default size' value.
    # Sets the minimum number of written replicas for objects in the
    # pool in order to acknowledge a write operation to the client. If
    # minimum is not met, Ceph will not acknowledge the write to the
    # client. This setting ensures a minimum number of replicas when
    # operating in degraded mode.
    # Type: 32-bit Integer
    # (Default: 0), which means no particular minimum. If 0, minimum is size - (size / 2).
    osd pool default min size  = 2

    ## Ensure you have a realistic number of placement groups. We recommend
    ## approximately 100 per OSD. E.g., total number of OSDs multiplied by 100
    ## divided by the number of replicas (i.e., osd pool default size). So for
    ## 10 OSDs and osd pool default size = 3, we'd recommend approximately
    ## (100 * 10) / 3 = 333

    # Description: The default number of placement groups for a pool. The
    #              default value is the same as pg_num with mkpool.
    # Type: 32-bit Integer
    # (Default: 8)
    osd pool default pg num    = 128

    # Description: The default number of placement groups for placement for a
    #              pool. The default value is the same as pgp_num with mkpool.
    #              PG and PGP should be equal (for now).
    # Type: 32-bit Integer
    # (Default: 8)
    osd pool default pgp num   = 128

    # The default CRUSH ruleset to use when creating a pool
    # Type: 32-bit Integer
    # (Default: 0)
    osd pool default crush rule = 0

    # The bucket type to use for chooseleaf in a CRUSH rule.
    # Uses ordinal rank rather than name.
    # Type: 32-bit Integer
    # (Default: 1) Typically a host containing one or more Ceph OSD Daemons.
    osd crush chooseleaf type = 1


### http://ceph.com/docs/master/rados/troubleshooting/log-and-debug/

    # The location of the logging file for your cluster.
    # Type: String
    # Required: No
    # Default: /var/log/ceph/$cluster-$name.log
    log file                   = /var/log/ceph/$cluster-$name.log

    # Determines if logging messages should appear in syslog.
    # Type: Boolean
    # Required: No
    # (Default: false)
    log to syslog              = true


### http://ceph.com/docs/master/rados/configuration/ms-ref/

    # Enable if you want your daemons to bind to IPv6 address instead of
    # IPv4 ones. (Not required if you specify a daemon or cluster IP.)
    # Type: Boolean
    # (Default: false)
    ms bind ipv6               = true

##################
## Monitors
## You need at least one. You need at least three if you want to
## tolerate any node failures. Always create an odd number.
[mon]
### http://ceph.com/docs/master/rados/configuration/mon-config-ref/
### http://ceph.com/docs/master/rados/configuration/mon-osd-interaction/

    # The IDs of initial monitors in a cluster during startup.
    # If specified, Ceph requires an odd number of monitors to form an
    # initial quorum (e.g., 3).
    # Type: String
    # (Default: None)
    mon initial members        = mycephhost

    mon host                   = cephhost01,cephhost02
    mon addr                   = 192.168.0.101,192.168.0.102

    # The monitor's data location
    # Default: /var/lib/ceph/mon/$cluster-$id
    mon data                   = /var/lib/ceph/mon/$name

    # The clock drift in seconds allowed between monitors.
    # Type: Float
    # (Default: .050)
    mon clock drift allowed    = .15

    # Exponential backoff for clock drift warnings
    # Type: Float
    # (Default: 5)
    mon clock drift warn backoff = 30    # Tell the monitor to backoff from this warning for 30 seconds

    # The percentage of disk space used before an OSD is considered full.
    # Type: Float
    # (Default: .95)
    mon osd full ratio         = .95

    # The percentage of disk space used before an OSD is considered nearfull.
    # Type: Float
    # (Default: .85)
    mon osd nearfull ratio     = .85

    # The number of seconds Ceph waits before marking a Ceph OSD
    # Daemon "down" and "out" if it doesn't respond.
    # Type: 32-bit Integer
    # (Default: 300)
    mon osd down out interval  = 300

    # The grace period in seconds before declaring unresponsive Ceph OSD
    # Daemons "down".
    # Type: 32-bit Integer
    # (Default: 900)
    mon osd report timeout          = 300

### http://ceph.com/docs/master/rados/troubleshooting/log-and-debug/

    # logging, for debugging monitor crashes, in order of
    # their likelihood of being helpful :)
    debug ms                   = 1
    debug mon                  = 20
    debug paxos                = 20
    debug auth                 = 20


[mon.alpha]
    host                       = alpha
    mon addr                   = 192.168.0.10:6789

[mon.beta]
    host                       = beta
    mon addr                   = 192.168.0.11:6789

[mon.gamma]
    host                       = gamma
    mon addr                   = 192.168.0.12:6789


##################
## Metadata servers
# You must deploy at least one metadata server to use CephFS. There is
# experimental support for running multiple metadata servers. Do not run
# multiple metadata servers in production.
[mds]
### http://ceph.com/docs/master/cephfs/mds-config-ref/

    # where the mds keeps it's secret encryption keys
    keyring                    = /var/lib/ceph/mds/$name/keyring

    # Determines whether a 'ceph-mds' daemon should poll and
    # replay the log of an active MDS (hot standby).
    # Type:  Boolean
    # (Default: false)
    mds standby replay          = true

    # mds logging to debug issues.
    debug ms                   = 1
    debug mds                  = 20
    debug journaler            = 20

    # The number of inodes to cache.
    # Type: 32-bit Integer
    # (Default: 100000)
    mds cache size             = 250000

[mds.alpha]
   host                       = alpha

[mds.beta]
   host                       = beta

##################
## osd
# You need at least one.  Two or more if you want data to be replicated.
# Define as many as you like.
[osd]
### http://ceph.com/docs/master/rados/configuration/osd-config-ref/

    # The path to the OSDs data.
    # You must create the directory when deploying Ceph.
    # You should mount a drive for OSD data at this mount point.
    # We do not recommend changing the default.
    # Type: String
    # Default: /var/lib/ceph/osd/$cluster-$id
    osd data                     = /var/lib/ceph/osd/$name

    ## You can change the number of recovery operations to speed up recovery
    ## or slow it down if your machines can't handle it

    # The number of active recovery requests per OSD at one time.
    # More requests will accelerate recovery, but the requests
    # places an increased load on the cluster.
    # Type: 32-bit Integer
    # (Default: 5)
    osd recovery max active      = 3

    # The maximum number of backfills allowed to or from a single OSD.
    # Type: 64-bit Integer
    # (Default: 10)
    osd max backfills            = 5

    # The maximum number of simultaneous scrub operations for a Ceph OSD Daemon.
    # Type: 32-bit Int
    # (Default: 1)
    osd max scrubs               = 2

    # You may add settings for ceph-deploy so that it will create and mount
    # the correct type of file system. Remove the comment `#` character for
    # the following settings and replace the values in parenthesis
    # with appropriate values, or leave the following settings commented
    # out to accept the default values.

    #osd mkfs type = {fs-type}
    #osd mkfs options {fs-type}   = {mkfs options}   # default for xfs is "-f"
    #osd mount options {fs-type}  = {mount options} # default mount option is "rw, noatime"
    osd mkfs type                = btrfs
    osd mount options btrfs      = noatime,nodiratime

    ## Ideally, make this a separate disk or partition.  A few
    ## hundred MB should be enough; more if you have fast or many
    ## disks.  You can use a file under the osd data dir if need be
    ## (e.g. /data/$name/journal), but it will be slower than a
    ## separate disk or partition.
    # The path to the OSD's journal. This may be a path to a file or a block
    # device (such as a partition of an SSD). If it is a file, you must
    # create the directory to contain it.
    # We recommend using a drive separate from the osd data drive.
    # Type: String
    # Default: /var/lib/ceph/osd/$cluster-$id/journal
    osd journal                  = /var/lib/ceph/osd/$name/journal

    # Check log files for corruption. Can be computationally expensive.
    # Type: Boolean
    # (Default: false)
    osd check for log corruption = true

### http://ceph.com/docs/master/rados/configuration/journal-ref/

    # The size of the journal in megabytes. If this is 0,
    # and the journal is a block device, the entire block device is used.
    # Since v0.54, this is ignored if the journal is a block device,
    # and the entire block device is used.
    # Type: 32-bit Integer
    # (Default: 5120)
    # Recommended: Begin with 1GB. Should be at least twice the product
    # of the expected speed multiplied by "filestore max sync interval".
    osd journal size             = 2048     ; journal size, in megabytes

    ## If you want to run the journal on a tmpfs, disable DirectIO
    # Enables direct i/o to the journal.
    # Requires "journal block align" set to "true".
    # Type: Boolean
    # Required: Yes when using aio.
    # (Default: true)
    journal dio                  = false

    # osd logging to debug osd issues, in order of likelihood of being helpful
    debug ms                     = 1
    debug osd                    = 20
    debug filestore              = 20
    debug journal                = 20

### http://ceph.com/docs/master/rados/configuration/filestore-config-ref/

    # The maximum interval in seconds for synchronizing the filestore.
    # Type: Double (optional)
    # (Default: 5)
    filestore max sync interval = 5

    # Enable snapshots for a btrfs filestore.
    # Type: Boolean
    # Required: No. Only used for btrfs.
    # (Default: true)
    filestore btrfs snap        = false

    # Enables the filestore flusher.
    # Type: Boolean
    # Required: No
    # (Default: false)
    filestore flusher            = true

    # Defines the maximum number of in progress operations the file store
    # accepts before blocking on queuing new operations.
    # Type: Integer
    # Required: No. Minimal impact on performance.
    # (Default: 500)
    filestore queue max ops      = 500

    ## Filestore and OSD settings can be tweak to achieve better performance

### http://ceph.com/docs/master/rados/configuration/filestore-config-ref/#misc

    # Min number of files in a subdir before merging into parent NOTE: A negative value means to disable subdir merging
    # Type: Integer
    # Required: No
    # Default:  10
    filestore merge threshold    = 10

    # filestore_split_multiple * abs(filestore_merge_threshold) * 16 is the maximum number of files in a subdirectory before splitting into child directories.
    # Type: Integer
    # Required: No
    # Default:  2
    filestore split multiple     = 2

    # The number of filesystem operation threads that execute in parallel.
    # Type: Integer
    # Required: No
    # Default:  2
    filestore op threads         = 4

    # The number of threads to service Ceph OSD Daemon operations. Set to 0 to disable it. Increasing the number may increase the request processing rate.
    # Type: 32-bit Integer
    # Default:  2
    osd op threads               = 2

    ## CRUSH

    # By default OSDs update their details (location, weight and root) on the CRUSH map during startup
    # Type: Boolean
    # Required: No;
    # (Default: true)
    osd crush update on start    = false

[osd.0]
    host                         = delta

[osd.1]
    host                         = epsilon

[osd.2]
    host                         = zeta

[osd.3]
    host                         = eta


##################
## client settings
[client]

### http://ceph.com/docs/master/rbd/rbd-config-ref/

    # Enable caching for RADOS Block Device (RBD).
    # Type: Boolean
    # Required: No
    # (Default: true)
    rbd cache                           = true

    # The RBD cache size in bytes.
    # Type: 64-bit Integer
    # Required: No
    # (Default: 32 MiB)
    ;rbd cache size                     = 33554432

    # The dirty limit in bytes at which the cache triggers write-back.
    # If 0, uses write-through caching.
    # Type: 64-bit Integer
    # Required: No
    # Constraint: Must be less than rbd cache size.
    # (Default: 24 MiB)
    rbd cache max dirty                = 25165824

    # The dirty target before the cache begins writing data to the data storage.
    # Does not block writes to the cache.
    # Type: 64-bit Integer
    # Required: No
    # Constraint: Must be less than rbd cache max dirty.
    # (Default: 16 MiB)
    rbd cache target dirty             = 16777216

    # The number of seconds dirty data is in the cache before writeback starts.
    # Type: Float
    # Required: No
    # (Default: 1.0)
    rbd cache max dirty age            = 1.0

    # Start out in write-through mode, and switch to write-back after the
    # first flush request is received. Enabling this is a conservative but
    # safe setting in case VMs running on rbd are too old to send flushes,
    # like the virtio driver in Linux before 2.6.32.
    # Type: Boolean
    # Required: No
    # (Default: true)
    rbd cache writethrough until flush = true

    # The Ceph admin socket allows you to query a daemon via a socket interface
    # From a client perspective this can be a virtual machine using librbd
    # Type: String
    # Required: No
    admin socket                       = /var/run/ceph/$cluster-$type.$id.$pid.$cctid.asok


##################
## radosgw client settings
[client.radosgw.gateway]

### http://ceph.com/docs/master/radosgw/config-ref/

    # Sets the location of the data files for Ceph Object Gateway.
    # You must create the directory when deploying Ceph.
    # We do not recommend changing the default.
    # Type: String
    # Default: /var/lib/ceph/radosgw/$cluster-$id
    rgw data                     = /var/lib/ceph/radosgw/$name

    # Client's hostname
    host                         =  ceph-radosgw

    # where the radosgw keeps it's secret encryption keys
    keyring                      = /etc/ceph/ceph.client.radosgw.keyring

    # FastCgiExternalServer uses this socket.
    # If you do not specify a socket path, Ceph Object Gateway will not run as an external server.
    # The path you specify here must be the same as the path specified in the rgw.conf file.
    # Type: String
    # Default: None
    rgw socket path              = /var/run/ceph/ceph.radosgw.gateway.fastcgi.sock

    # The location of the logging file for your radosgw.
    # Type: String
    # Required: No
    # Default: /var/log/ceph/$cluster-$name.log
    log file                     = /var/log/ceph/client.radosgw.gateway.log

    # Enable 100-continue if it is operational.
    # Type: Boolean
    # Default:  true
    rgw print continue           = false

    # The DNS name of the served domain.
    # Type: String
    # Default:  None
    rgw dns name                 = radosgw.ceph.internal