Clustering - Agglomerative clustering - Práctica

Índice de contenido

python

import pandas as pd
from sklearn.cluster import AgglomerativeClustering

python

file_datapath = "C:/Users/user/Desktop/water-treatment.data"
df = pd.read_csv(file_datapath)
df

	D-1/3/90	44101	1.50	7.8	?	407	166	66.3	4.5	2110	...	2000	?.4	58.8	95.5	?.5	70.0	?.6	79.4	87.3	99.6
0	D-2/3/90	39024	3.00	7.7	?	443	214	69.2	6.5	2660	...	2590	?	60.7	94.8	?	80.8	?	79.5	92.1	100
1	D-4/3/90	32229	5.00	7.6	?	528	186	69.9	3.4	1666	...	1888	?	58.2	95.6	?	52.9	?	75.8	88.7	98.5
2	D-5/3/90	35023	3.50	7.9	205	588	192	65.6	4.5	2430	...	1840	33.1	64.2	95.3	87.3	72.3	90.2	82.3	89.6	100
3	D-6/3/90	36924	1.50	8.0	242	496	176	64.8	4.0	2110	...	2120	?	62.7	95.6	?	71.0	92.1	78.2	87.5	99.5
4	D-7/3/90	38572	3.00	7.8	202	372	186	68.8	4.5	1644	...	1764	?	59.7	96.5	86.7	78.3	90.1	73.1	84.9	100
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
521	D-26/8/91	32723	0.16	7.7	93	252	176	56.8	2.3	894	...	942	?	62.3	93.3	69.8	75.9	79.6	78.6	96.6	99.6
522	D-27/8/91	33535	0.32	7.8	192	346	172	68.6	4.0	988	...	950	?	58.3	97.8	83.0	59.1	91.1	74.6	90.7	100
523	D-28/8/91	32922	0.30	7.4	139	367	180	64.4	3.0	1060	...	1136	?	65.0	97.1	76.2	66.4	82.0	77.1	88.9	99
524	D-29/8/91	32190	0.30	7.3	200	545	258	65.1	4.0	1260	...	1326	39.8	65.9	97.1	81.7	70.9	89.5	87.0	89.5	99.8
525	D-30/8/91	30488	0.21	7.5	152	300	132	69.7	?	1073	...	1224	?	69.5	?	81.7	76.4	?	81.7	86.4	?

526 rows × 39 columns

python

df.set_index(df["D-1/3/90"], drop=True, inplace=True)

python

df.drop("D-1/3/90", axis=1, inplace = True)

python

df

	44101	1.50	7.8	?	407	166	66.3	4.5	2110	7.9	...	2000	?.4	58.8	95.5	?.5	70.0	?.6	79.4	87.3	99.6
D-1/3/90
D-2/3/90	39024	3.00	7.7	?	443	214	69.2	6.5	2660	7.7	...	2590	?	60.7	94.8	?	80.8	?	79.5	92.1	100
D-4/3/90	32229	5.00	7.6	?	528	186	69.9	3.4	1666	7.7	...	1888	?	58.2	95.6	?	52.9	?	75.8	88.7	98.5
D-5/3/90	35023	3.50	7.9	205	588	192	65.6	4.5	2430	7.8	...	1840	33.1	64.2	95.3	87.3	72.3	90.2	82.3	89.6	100
D-6/3/90	36924	1.50	8.0	242	496	176	64.8	4.0	2110	7.9	...	2120	?	62.7	95.6	?	71.0	92.1	78.2	87.5	99.5
D-7/3/90	38572	3.00	7.8	202	372	186	68.8	4.5	1644	7.8	...	1764	?	59.7	96.5	86.7	78.3	90.1	73.1	84.9	100
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
D-26/8/91	32723	0.16	7.7	93	252	176	56.8	2.3	894	7.7	...	942	?	62.3	93.3	69.8	75.9	79.6	78.6	96.6	99.6
D-27/8/91	33535	0.32	7.8	192	346	172	68.6	4.0	988	7.8	...	950	?	58.3	97.8	83.0	59.1	91.1	74.6	90.7	100
D-28/8/91	32922	0.30	7.4	139	367	180	64.4	3.0	1060	7.5	...	1136	?	65.0	97.1	76.2	66.4	82.0	77.1	88.9	99
D-29/8/91	32190	0.30	7.3	200	545	258	65.1	4.0	1260	7.4	...	1326	39.8	65.9	97.1	81.7	70.9	89.5	87.0	89.5	99.8
D-30/8/91	30488	0.21	7.5	152	300	132	69.7	?	1073	7.4	...	1224	?	69.5	?	81.7	76.4	?	81.7	86.4	?

526 rows × 38 columns

python

df.dtypes

44101     object
1.50      object
7.8      float64
?         object
407       object
166       object
66.3      object
4.5       object
2110       int64
7.9      float64
?.1       object
228        int64
70.2      object
5.5       object
2120       int64
7.9.1    float64
?.2       object
280       object
94        object
72.3      object
0.3       object
2010       int64
7.3       object
?.3       object
84        object
21        object
81.0      object
0.02      object
2000      object
?.4       object
58.8      object
95.5      object
?.5       object
70.0      object
?.6       object
79.4      object
87.3      object
99.6      object
dtype: object

python

df.replace("?", "", inplace=True)

python

df

	44101	1.50	7.8	?	407	166	66.3	4.5	2110	7.9	...	2000	?.4	58.8	95.5	?.5	70.0	?.6	79.4	87.3	99.6
D-1/3/90
D-2/3/90	39024	3.00	7.7		443	214	69.2	6.5	2660	7.7	...	2590		60.7	94.8		80.8		79.5	92.1	100
D-4/3/90	32229	5.00	7.6		528	186	69.9	3.4	1666	7.7	...	1888		58.2	95.6		52.9		75.8	88.7	98.5
D-5/3/90	35023	3.50	7.9	205	588	192	65.6	4.5	2430	7.8	...	1840	33.1	64.2	95.3	87.3	72.3	90.2	82.3	89.6	100
D-6/3/90	36924	1.50	8.0	242	496	176	64.8	4.0	2110	7.9	...	2120		62.7	95.6		71.0	92.1	78.2	87.5	99.5
D-7/3/90	38572	3.00	7.8	202	372	186	68.8	4.5	1644	7.8	...	1764		59.7	96.5	86.7	78.3	90.1	73.1	84.9	100
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
D-26/8/91	32723	0.16	7.7	93	252	176	56.8	2.3	894	7.7	...	942		62.3	93.3	69.8	75.9	79.6	78.6	96.6	99.6
D-27/8/91	33535	0.32	7.8	192	346	172	68.6	4.0	988	7.8	...	950		58.3	97.8	83.0	59.1	91.1	74.6	90.7	100
D-28/8/91	32922	0.30	7.4	139	367	180	64.4	3.0	1060	7.5	...	1136		65.0	97.1	76.2	66.4	82.0	77.1	88.9	99
D-29/8/91	32190	0.30	7.3	200	545	258	65.1	4.0	1260	7.4	...	1326	39.8	65.9	97.1	81.7	70.9	89.5	87.0	89.5	99.8
D-30/8/91	30488	0.21	7.5	152	300	132	69.7		1073	7.4	...	1224		69.5		81.7	76.4		81.7	86.4

526 rows × 38 columns

python

df = df.apply(pd.to_numeric)

python

df

	44101	1.50	7.8	?	407	166	66.3	4.5	2110	7.9	...	2000	?.4	58.8	95.5	?.5	70.0	?.6	79.4	87.3	99.6
D-1/3/90
D-2/3/90	39024.0	3.00	7.7	NaN	443.0	214.0	69.2	6.5	2660	7.7	...	2590.0	NaN	60.7	94.8	NaN	80.8	NaN	79.5	92.1	100.0
D-4/3/90	32229.0	5.00	7.6	NaN	528.0	186.0	69.9	3.4	1666	7.7	...	1888.0	NaN	58.2	95.6	NaN	52.9	NaN	75.8	88.7	98.5
D-5/3/90	35023.0	3.50	7.9	205.0	588.0	192.0	65.6	4.5	2430	7.8	...	1840.0	33.1	64.2	95.3	87.3	72.3	90.2	82.3	89.6	100.0
D-6/3/90	36924.0	1.50	8.0	242.0	496.0	176.0	64.8	4.0	2110	7.9	...	2120.0	NaN	62.7	95.6	NaN	71.0	92.1	78.2	87.5	99.5
D-7/3/90	38572.0	3.00	7.8	202.0	372.0	186.0	68.8	4.5	1644	7.8	...	1764.0	NaN	59.7	96.5	86.7	78.3	90.1	73.1	84.9	100.0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
D-26/8/91	32723.0	0.16	7.7	93.0	252.0	176.0	56.8	2.3	894	7.7	...	942.0	NaN	62.3	93.3	69.8	75.9	79.6	78.6	96.6	99.6
D-27/8/91	33535.0	0.32	7.8	192.0	346.0	172.0	68.6	4.0	988	7.8	...	950.0	NaN	58.3	97.8	83.0	59.1	91.1	74.6	90.7	100.0
D-28/8/91	32922.0	0.30	7.4	139.0	367.0	180.0	64.4	3.0	1060	7.5	...	1136.0	NaN	65.0	97.1	76.2	66.4	82.0	77.1	88.9	99.0
D-29/8/91	32190.0	0.30	7.3	200.0	545.0	258.0	65.1	4.0	1260	7.4	...	1326.0	39.8	65.9	97.1	81.7	70.9	89.5	87.0	89.5	99.8
D-30/8/91	30488.0	0.21	7.5	152.0	300.0	132.0	69.7	NaN	1073	7.4	...	1224.0	NaN	69.5	NaN	81.7	76.4	NaN	81.7	86.4	NaN

526 rows × 38 columns

python

df.mean()

44101    37213.035433
1.50         2.360707
7.8          7.810076
?          188.714286
407        406.898077
166        227.561905
66.3        61.383689
4.5          4.594012
2110      1477.420152
7.9          7.829848
?.1        206.207392
228        254.001901
70.2        60.351262
5.5          5.032669
2120      1494.847909
7.9.1        7.811787
?.2        122.348697
280        274.034816
94          94.225191
72.3        72.969786
0.3          0.416966
2010      1489.581749
7.3          7.710667
?.3         19.988095
84          87.301181
21          22.238004
81.0        80.150098
0.02         0.037108
2000      1493.855238
?.4         39.085806
58.8        58.518199
95.5        90.544289
?.5         83.448049
70.0        67.813000
?.6         89.013646
79.4        77.853493
87.3        88.966602
99.6        99.085253
dtype: float64

python

df = df.fillna(df.mean())
df

	44101	1.50	7.8	?	407	166	66.3	4.5	2110	7.9	...	2000	?.4	58.8	95.5	?.5	70.0	?.6	79.4	87.3	99.6
D-1/3/90
D-2/3/90	39024.0	3.00	7.7	188.714286	443.0	214.0	69.2	6.500000	2660	7.7	...	2590.0	39.085806	60.7	94.800000	83.448049	80.8	89.013646	79.5	92.1	100.000000
D-4/3/90	32229.0	5.00	7.6	188.714286	528.0	186.0	69.9	3.400000	1666	7.7	...	1888.0	39.085806	58.2	95.600000	83.448049	52.9	89.013646	75.8	88.7	98.500000
D-5/3/90	35023.0	3.50	7.9	205.000000	588.0	192.0	65.6	4.500000	2430	7.8	...	1840.0	33.100000	64.2	95.300000	87.300000	72.3	90.200000	82.3	89.6	100.000000
D-6/3/90	36924.0	1.50	8.0	242.000000	496.0	176.0	64.8	4.000000	2110	7.9	...	2120.0	39.085806	62.7	95.600000	83.448049	71.0	92.100000	78.2	87.5	99.500000
D-7/3/90	38572.0	3.00	7.8	202.000000	372.0	186.0	68.8	4.500000	1644	7.8	...	1764.0	39.085806	59.7	96.500000	86.700000	78.3	90.100000	73.1	84.9	100.000000
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
D-26/8/91	32723.0	0.16	7.7	93.000000	252.0	176.0	56.8	2.300000	894	7.7	...	942.0	39.085806	62.3	93.300000	69.800000	75.9	79.600000	78.6	96.6	99.600000
D-27/8/91	33535.0	0.32	7.8	192.000000	346.0	172.0	68.6	4.000000	988	7.8	...	950.0	39.085806	58.3	97.800000	83.000000	59.1	91.100000	74.6	90.7	100.000000
D-28/8/91	32922.0	0.30	7.4	139.000000	367.0	180.0	64.4	3.000000	1060	7.5	...	1136.0	39.085806	65.0	97.100000	76.200000	66.4	82.000000	77.1	88.9	99.000000
D-29/8/91	32190.0	0.30	7.3	200.000000	545.0	258.0	65.1	4.000000	1260	7.4	...	1326.0	39.800000	65.9	97.100000	81.700000	70.9	89.500000	87.0	89.5	99.800000
D-30/8/91	30488.0	0.21	7.5	152.000000	300.0	132.0	69.7	4.594012	1073	7.4	...	1224.0	39.085806	69.5	90.544289	81.700000	76.4	89.013646	81.7	86.4	99.085253

526 rows × 38 columns

python

df.dtypes

44101    float64
1.50     float64
7.8      float64
?        float64
407      float64
166      float64
66.3     float64
4.5      float64
2110       int64
7.9      float64
?.1      float64
228        int64
70.2     float64
5.5      float64
2120       int64
7.9.1    float64
?.2      float64
280      float64
94       float64
72.3     float64
0.3      float64
2010       int64
7.3      float64
?.3      float64
84       float64
21       float64
81.0     float64
0.02     float64
2000     float64
?.4      float64
58.8     float64
95.5     float64
?.5      float64
70.0     float64
?.6      float64
79.4     float64
87.3     float64
99.6     float64
dtype: object

python

agc = AgglomerativeClustering(n_clusters=13, 
                              affinity='euclidean', 
                              linkage='ward', 
                              distance_threshold=None)

python

agc.fit(df)

AgglomerativeClustering(n_clusters=13)

python

agc.labels_

array([ 8,  3,  1,  7, 11,  8,  1,  4,  8,  0,  0,  0,  8,  8,  0, 11,  1,
        7,  8,  0,  5,  0,  8,  8,  8,  0,  0,  4,  7, 11, 11,  1,  7,  3,
       11,  7,  1,  1,  1,  7,  0, 11,  8,  8,  7,  8,  0, 12,  8,  7,  1,
        3,  6,  7, 11,  1,  4,  3,  4,  7,  5,  0,  8,  8, 11,  5,  5,  5,
        0,  8,  3,  8,  8,  5,  1,  0,  9, 11,  7,  3,  1,  8, 12,  0, 11,
        0,  0,  1,  7,  1,  1,  1,  1, 11, 11,  0, 11,  1,  1,  6, 11, 11,
       11, 11,  6, 11,  8,  0,  0, 11, 11,  7,  7,  1,  1,  3,  7,  0,  9,
        0,  0,  5,  5, 12, 12,  8,  9,  9,  5, 12,  9,  7,  1,  8,  0,  1,
        4,  1, 11,  0,  8,  7,  5,  9, 11,  1,  4,  6,  4,  8,  7,  7,  7,
        3,  7,  8,  0,  7,  1,  1,  4,  7,  1,  3,  3,  3,  4,  6,  1,  1,
        3,  1,  6,  7,  3,  0,  0,  8, 11,  7,  7,  0, 11,  7,  7,  8,  2,
        5,  9, 12,  0, 12,  7,  8, 10,  5,  5, 11,  0,  3,  2,  9,  9,  8,
       12,  5,  0,  0,  5,  0,  1, 11,  4, 11, 11,  1,  1,  6,  3,  1,  0,
        8,  1,  8,  0,  4,  4,  6,  6,  3,  1,  6,  6,  3,  4,  3,  1,  7,
        1,  7,  7,  7,  6,  4,  1,  1,  3,  4, 12,  0, 11,  0,  0,  0, 12,
        5,  0, 12,  5,  0,  4,  4,  1,  5,  5,  8,  6,  4,  6,  0,  5,  3,
        6,  6,  5,  9,  7,  1,  8, 12,  5, 12,  0, 11,  1,  9,  9,  5, 12,
       12,  0,  5, 10,  8,  8, 12,  5,  5,  5,  6,  3,  8,  8, 12,  0,  0,
        7,  7,  1,  7,  4,  3,  6,  4,  4,  3,  1,  3,  5, 12,  0,  8,  1,
        3, 11,  4,  1,  1,  7,  1,  1,  4,  4,  3,  1,  3,  1,  1,  7,  7,
        4, 11,  6,  4,  4,  3,  6,  3,  8,  1,  1,  1,  8, 11,  1,  0, 12,
        6,  0,  5,  5, 12,  7,  4,  1,  1,  7,  1,  7,  6,  3,  3,  3,  1,
       12,  0,  1,  7,  3,  1, 10, 10,  5, 10,  5,  1,  1,  3,  3,  3,  5,
        8, 11,  7,  7,  7,  0, 11,  1,  1,  1,  1,  1,  1,  1,  0,  6,  3,
        1,  1,  8,  6,  3,  3,  3,  1,  1, 11,  4,  7,  1,  1,  7,  0,  6,
        1,  3,  1,  1,  3,  3,  6,  1,  3,  1,  7,  7,  7,  7,  7,  7,  7,
        7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  7,  3,  4,  4,  1,  1,  1,
        4,  1,  3,  3,  7,  3,  1,  4,  1,  7,  3,  3,  1,  1,  1,  3,  4,
        3,  1,  1,  3,  1,  4,  4,  1,  4,  1,  1,  0,  1,  4,  3,  1,  3,
        6,  7,  1,  1,  4,  1,  4,  4,  4,  6,  6,  4,  4,  4,  4,  4,  2,
        3,  6,  3,  6,  6,  3,  4,  3,  6,  6,  4,  3,  3,  3,  3,  4],
      dtype=int64)

python

df["cluster_id"] = agc.labels_
df

	44101	1.50	7.8	?	407	166	66.3	4.5	2110	7.9	...	?.4	58.8	95.5	?.5	70.0	?.6	79.4	87.3	99.6	cluster_id
D-1/3/90
D-2/3/90	39024.0	3.00	7.7	188.714286	443.0	214.0	69.2	6.500000	2660	7.7	...	39.085806	60.7	94.800000	83.448049	80.8	89.013646	79.5	92.1	100.000000	8
D-4/3/90	32229.0	5.00	7.6	188.714286	528.0	186.0	69.9	3.400000	1666	7.7	...	39.085806	58.2	95.600000	83.448049	52.9	89.013646	75.8	88.7	98.500000	3
D-5/3/90	35023.0	3.50	7.9	205.000000	588.0	192.0	65.6	4.500000	2430	7.8	...	33.100000	64.2	95.300000	87.300000	72.3	90.200000	82.3	89.6	100.000000	1
D-6/3/90	36924.0	1.50	8.0	242.000000	496.0	176.0	64.8	4.000000	2110	7.9	...	39.085806	62.7	95.600000	83.448049	71.0	92.100000	78.2	87.5	99.500000	7
D-7/3/90	38572.0	3.00	7.8	202.000000	372.0	186.0	68.8	4.500000	1644	7.8	...	39.085806	59.7	96.500000	86.700000	78.3	90.100000	73.1	84.9	100.000000	11
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
D-26/8/91	32723.0	0.16	7.7	93.000000	252.0	176.0	56.8	2.300000	894	7.7	...	39.085806	62.3	93.300000	69.800000	75.9	79.600000	78.6	96.6	99.600000	3
D-27/8/91	33535.0	0.32	7.8	192.000000	346.0	172.0	68.6	4.000000	988	7.8	...	39.085806	58.3	97.800000	83.000000	59.1	91.100000	74.6	90.7	100.000000	3
D-28/8/91	32922.0	0.30	7.4	139.000000	367.0	180.0	64.4	3.000000	1060	7.5	...	39.085806	65.0	97.100000	76.200000	66.4	82.000000	77.1	88.9	99.000000	3
D-29/8/91	32190.0	0.30	7.3	200.000000	545.0	258.0	65.1	4.000000	1260	7.4	...	39.800000	65.9	97.100000	81.700000	70.9	89.500000	87.0	89.5	99.800000	3
D-30/8/91	30488.0	0.21	7.5	152.000000	300.0	132.0	69.7	4.594012	1073	7.4	...	39.085806	69.5	90.544289	81.700000	76.4	89.013646	81.7	86.4	99.085253	4

526 rows × 39 columns

python

df.loc[df["cluster_id"] == 1]

	44101	1.50	7.8	?	407	166	66.3	4.5	2110	7.9	...	?.4	58.8	95.5	?.5	70.0	?.6	79.4	87.3	99.6	cluster_id
D-1/3/90
D-5/3/90	35023.0	3.50	7.9	205.0	588.000000	192.0	65.6	4.5	2430	7.8	...	33.100000	64.2	95.3	87.300000	72.300	90.2	82.300000	89.600000	100.0	1
D-9/3/90	36107.0	5.00	7.7	215.0	489.000000	334.0	40.7	6.0	1613	7.6	...	39.085806	70.4	95.6	90.600000	53.700	92.1	66.900000	94.600000	100.0	1
D-21/3/90	35791.0	1.20	7.8	277.0	466.000000	246.0	63.4	4.0	1556	7.7	...	39.085806	39.6	78.3	85.500000	70.400	91.3	73.400000	89.400000	99.4	1
D-8/2/90	34193.0	2.00	8.0	166.0	396.000000	176.0	70.5	4.0	1265	8.0	...	7.300000	52.1	96.4	84.200000	71.200	84.3	73.200000	82.400000	100.0	1
D-14/2/90	35636.0	1.20	8.0	203.0	469.000000	264.0	65.2	5.2	1489	8.1	...	39.085806	67.1	95.3	85.200000	73.100	88.7	79.300000	85.600000	99.0	1
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
D-16/10/91	34820.0	0.25	8.1	185.0	439.000000	256.0	56.3	7.5	2210	7.9	...	32.800000	67.7	96.4	72.700000	81.000	82.2	89.300000	91.800000	99.7	1
D-19/10/91	34408.0	0.25	8.0	174.0	442.000000	268.0	57.5	5.7	1306	7.9	...	32.800000	68.2	94.4	83.500000	80.400	88.5	87.800000	93.300000	99.6	1
D-24/10/91	34364.0	1.20	7.9	191.0	406.898077	184.0	73.9	6.5	1384	7.8	...	40.600000	59.8	92.0	89.500000	67.813	93.7	77.853493	91.800000	98.5	1
D-25/10/91	35400.0	0.70	7.6	156.0	364.000000	194.0	63.9	5.5	1680	7.6	...	47.300000	61.3	94.0	76.400000	67.813	86.5	82.400000	90.700000	99.8	1
D-27/10/91	35573.0	7.30	7.6	176.0	333.000000	178.0	64.0	3.5	1627	7.7	...	39.085806	40.4	95.0	83.448049	72.900	90.9	79.900000	88.966602	98.6	1

103 rows × 39 columns