Performance Comparison of Multiple Array Deduplication Methods
# Performance Comparison of Multiple Array Deduplication Methods
# Test Template
// Create an array from 1 to 1,000,000. Array.from is ES6 syntax
let arr1 = Array.from(new Array(1000000), (x, index) => {
return index
})
let arr2 = Array.from(new Array(500000), (x, index) => {
return index + index
})
let start = new Date().getTime()
console.log('Starting array deduplication')
// Array deduplication
function distinct(a, b) {
let arr = a.concat(b);
// Deduplication method
}
console.log('Length after deduplication', distinct(arr1, arr2).length)
let end = new Date().getTime()
console.log('Time elapsed', end - start + 'ms')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Test Code
// Create an array from 1 to 1,000,000. Array.from is ES6 syntax
let arr1 = Array.from(new Array(1000000), (x, index) => {
return index
})
let arr2 = Array.from(new Array(500000), (x, index) => {
return index + index
})
let start = new Date().getTime()
console.log('Starting array deduplication')
// Array deduplication
function distinct(a, b) {
let arr = a.concat(b);
// Method 1, takes about 11675ms, ~11s
// return arr.filter((item, index) => {
// return arr.indexOf(item) === index
// })
// Method 2, takes about 22851ms, ~22s, worst performance
// for (let i = 0, len = arr.length; i < len; i++) {
// for (let j = i + 1; j < len; j++) {
// if (arr[i] == arr[j]) {
// arr.splice(j, 1);
// // splice changes the array length, so both len and index j must be decremented
// len--;
// j--;
// }
// }
// }
// return arr
// Method 3, takes about 12789ms, ~12s, comparable to Method 1
// let result = []
// for (let i of arr) {
// !result.includes(i) && result.push(i)
// }
// return result
// Method 4, takes about 23ms, best performance among ES5 methods
// arr = arr.sort()
// let result = [arr[0]]
// for (let i = 1, len = arr.length; i < len; i++) {
// arr[i] !== arr[i - 1] && result.push(arr[i])
// }
// return result
// Method 5, ES6 Set data structure, takes about 20ms, high performance, concise code
// return Array.from(new Set([...a, ...b]))
// Method 6, takes about 16ms, best performance among all methods! (4x faster than Method 5 with tens of millions of entries, for...of is ES6 syntax)
let result = []
let obj = {}
for (let i of arr) {
if (!obj[i]) {
result.push(i)
obj[i] = 1
}
}
return result
}
console.log('Length after deduplication', distinct(arr1, arr2).length)
let end = new Date().getTime()
console.log('Time elapsed', end - start + 'ms')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
# Conclusion
The best-performing array deduplication method in the ES5 standard is:
// Takes about 23ms
arr = arr.sort()
let result = [arr[0]]
for (let i = 1, len = arr.length; i < len; i++) {
arr[i] !== arr[i - 1] && result.push(arr[i])
}
return result
1
2
3
4
5
6
7
2
3
4
5
6
7
The best-performing array deduplication method in the ES6 standard is:
// Takes about 16ms (4x faster than the Set approach with tens of millions of entries, for...of is ES6 syntax)
let result = []
let obj = {}
for (let i of arr) {
if (!obj[i]) {
result.push(i)
obj[i] = 1
}
}
return result
1
2
3
4
5
6
7
8
9
10
2
3
4
5
6
7
8
9
10
The method that offers both concise code and relatively high performance is:
// Takes about 20ms, high performance, concise code
return Array.from(new Set([...a, ...b]))
1
2
2
Edit (opens new window)
Last Updated: 2026/03/21, 12:14:36